| 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221 |
1×
1×
1×
114×
106×
300×
300×
106×
106×
102×
24×
78×
78×
260×
260×
30×
78×
10×
68×
4×
16×
72×
2×
72×
1×
392×
392×
989×
989×
92×
897×
392×
1×
284×
824×
824×
824×
1×
824×
32×
32×
74×
74×
32×
792×
792×
792×
2×
790×
2×
788×
788×
1×
866×
866×
1×
104×
284×
1×
157×
136×
136×
136×
116×
2×
114×
22×
114×
104×
20×
56×
104×
2×
6×
104×
98×
104×
10×
| 'use strict';
var _ = require('underscore'),
constants = require('./constants.json'),
path = require('doc-path'),
promise = require('bluebird');
var options = {}; // Initialize the options - this will be populated when the json2csv function is called.
/**
* Retrieve the headings for all documents and return it.
* This checks that all documents have the same schema.
* @param data
* @returns {promise}
*/
var generateHeading = function(data) {
if (options.KEYS) { return promise.resolve(options.KEYS); }
var keys = _.map(data, function (document, indx) { // for each key
Eif (_.isObject(document)) {
// if the data at the key is a document, then we retrieve the subHeading starting with an empty string heading and the doc
return generateDocumentHeading('', document);
}
});
var uniqueKeys = [];
// If the user wants to check for the same schema:
if (options.CHECK_SCHEMA_DIFFERENCES) {
// Check for a consistent schema that does not require the same order:
// if we only have one document - then there is no possibility of multiple schemas
if (keys && keys.length <= 1) {
return promise.resolve(_.flatten(keys) || []);
}
// else - multiple documents - ensure only one schema (regardless of field ordering)
var firstDocSchema = _.flatten(keys[0]),
schemaDifferences = 0;
_.each(keys, function (keyList) {
// If there is a difference between the schemas, increment the counter of schema inconsistencies
var diff = _.difference(firstDocSchema, _.flatten(keyList));
if (!_.isEqual(diff, [])) {
schemaDifferences++;
}
});
// If there are schema inconsistencies, throw a schema not the same error
if (schemaDifferences) {
return promise.reject(new Error(constants.Errors.json2csv.notSameSchema));
}
uniqueKeys = _.flatten(keys[0]);
} else {
// Otherwise, we do not care if the schemas are different, so we should merge them via union:
_.each(keys, function (keyList) {
uniqueKeys = _.union(uniqueKeys, _.flatten(keyList));
});
}
if (options.SORT_HEADER) {
uniqueKeys.sort();
}
return promise.resolve(uniqueKeys);
};
/**
* Takes the parent heading and this doc's data and creates the subdocument headings (string)
* @param heading
* @param data
* @returns {Array}
*/
var generateDocumentHeading = function(heading, data) {
var keyName = ''; // temporary variable to aid in determining the heading - used to generate the 'nested' headings
var documentKeys = _.map(_.keys(data), function (currentKey) {
// If the given heading is empty, then we set the heading to be the subKey, otherwise set it as a nested heading w/ a dot
keyName = heading ? heading + '.' + currentKey : currentKey;
// If we have another nested document, recur on the sub-document to retrieve the full key name
if (_.isObject(data[currentKey]) && !_.isNull(data[currentKey]) && !_.isArray(data[currentKey]) && _.keys(data[currentKey]).length) {
return generateDocumentHeading(keyName, data[currentKey]);
}
// Otherwise return this key name since we don't have a sub document
return keyName;
});
return documentKeys; // Return the headings in an array
};
/**
* Convert the given data with the given keys
* @param data
* @param keys
* @returns {Array}
*/
var
convertData = function (data, keys) {
// Reduce each key in the data to its CSV value
return _.reduce(keys, function (output, key) {
// Retrieve the appropriate field data
var fieldData = path.evaluatePath(data, key);
if (_.isUndefined(fieldData)) { fieldData = options.EMPTY_FIELD_VALUE; }
// Add the CSV representation of the data at the key in the document to the output array
return output.concat(convertField(fieldData));
}, []);
};
/**
* Convert the given value to the CSV representation of the value
* @param value
* @param output
*/
var convertField = function (value) {
if (_.isArray(value)) { // We have an array of values
var result = [];
value.forEach(function(item) {
Iif (_.isObject(item)) {
// use JSON stringify to convert objects in arrays, otherwise toString() will just return [object Object]
result.push(JSON.stringify(item));
} else {
result.push(convertValue(item));
}
});
return options.DELIMITER.WRAP + '[' + result.join(options.DELIMITER.ARRAY) + ']' + options.DELIMITER.WRAP;
} else Iif (_.isDate(value)) { // If we have a date
return options.DELIMITER.WRAP + convertValue(value) + options.DELIMITER.WRAP;
} else Iif (_.isObject(value)) { // If we have an object
return options.DELIMITER.WRAP + convertData(value, _.keys(value)) + options.DELIMITER.WRAP; // Push the recursively generated CSV
} else if (_.isNumber(value)) { // If we have a number (avoids 0 being converted to '')
return options.DELIMITER.WRAP + convertValue(value) + options.DELIMITER.WRAP;
} else if (_.isBoolean(value)) { // If we have a boolean (avoids false being converted to '')
return options.DELIMITER.WRAP + convertValue(value) + options.DELIMITER.WRAP;
}
value = options.DELIMITER.WRAP && value ? value.replace(new RegExp(options.DELIMITER.WRAP, 'g'), "\\"+options.DELIMITER.WRAP) : value;
return options.DELIMITER.WRAP + convertValue(value) + options.DELIMITER.WRAP; // Otherwise push the current value
};
var convertValue = function (val) {
// Convert to string
val = _.isNull(val) || _.isUndefined(val) ? '' : val.toString();
// Trim, if necessary, and return the correct value
return options.TRIM_FIELD_VALUES ? val.trim() : val;
};
/**
* Generate the CSV representing the given data.
* @param data
* @param headingKeys
* @returns {*}
*/
var generateCsv = function (data, headingKeys) {
// Reduce each JSON document in data to a CSV string and append it to the CSV accumulator
return [headingKeys].concat(_.reduce(data, function (csv, doc) {
return csv += convertData(doc, headingKeys).join(options.DELIMITER.FIELD) + options.DELIMITER.EOL;
}, ''));
};
module.exports = {
/**
* Internally exported json2csv function
* Takes options as a document, data as a JSON document array, and a callback that will be used to report the results
* @param opts Object options object
* @param data String csv string
* @param callback Function callback function
*/
json2csv: function (opts, data, callback) {
// If a callback wasn't provided, throw an error
if (!callback) { throw new Error(constants.Errors.callbackRequired); }
// Shouldn't happen, but just in case
Iif (!opts) { return callback(new Error(constants.Errors.optionsRequired)); }
options = opts; // Options were passed, set the global options value
// If we don't receive data, report an error
if (!data) { return callback(new Error(constants.Errors.json2csv.cannotCallJson2CsvOn + data + '.')); }
// If the data was not a single document or an array of documents
if (!_.isObject(data)) {
return callback(new Error(constants.Errors.json2csv.dataNotArrayOfDocuments)); // Report the error back to the caller
}
// Single document, not an array
else if (_.isObject(data) && !data.length) {
data = [data]; // Convert to an array of the given document
}
// Retrieve the heading and then generate the CSV with the keys that are identified
generateHeading(data)
.then(_.partial(generateCsv, data))
.spread(function (csvHeading, csvData) {
// If the fields are supposed to be wrapped... (only perform this if we are actually prepending the header)
if (options.DELIMITER.WRAP && options.PREPEND_HEADER) {
csvHeading = _.map(csvHeading, function(headingKey) {
return options.DELIMITER.WRAP + headingKey + options.DELIMITER.WRAP;
});
}
if (options.TRIM_HEADER_FIELDS) {
csvHeading = _.map(csvHeading, function (headingKey) {
return headingKey.trim();
});
}
// If we are prepending the header, then join the csvHeading fields
if (options.PREPEND_HEADER) {
csvHeading = csvHeading.join(options.DELIMITER.FIELD);
}
// If we are prepending the header, then join the header and data by EOL, otherwise just return the data
return callback(null, options.PREPEND_HEADER ? csvHeading + options.DELIMITER.EOL + csvData : csvData);
})
.catch(function (err) {
return callback(err);
});
}
};
|