Skip to content

Commit

Permalink
Merge branch 'main' into 169-support-in-memory-zip-files-in-the-schem…
Browse files Browse the repository at this point in the history 
…a-validation-api
  • Loading branch information
@mohit-s96
mohit-s96 committed Oct 4, 2024
2 parents 6e274c4 + f5aed4b commit 37531ad
Show file tree
Hide file tree
Showing 9 changed files with 355 additions and 250 deletions.
2 changes: 1 addition & 1 deletion common.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,7 +570,7 @@ const readZipFileContents = pathOrBuffer => {

readStream.on('end', () => {
const contents = Buffer.concat(chunks).toString('utf8');
result[entry.fileName.slice(entry.fileName.lastIndexOf('/') + 1, entry.fileName.length)] = contents;
result[entry.fileName] = contents;
// Move to the next entry.
zipfile.readEntry();
});
Expand Down
2 changes: 1 addition & 1 deletion lib/replication/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -818,7 +818,7 @@ const writeAnalyticsReports = async ({ outputPath, version, serviceRootUri, repl

console.log(`Response info written to ${resolvedPath}`);

const resolvedAvailabilityPath = resolveFilePathSync({ outputPath, AVAILABILITY_REPORT_FILENAME });
const resolvedAvailabilityPath = resolveFilePathSync({ outputPath, filename: AVAILABILITY_REPORT_FILENAME });

// write DA report
await writeFile(
Expand Down
2 changes: 1 addition & 1 deletion lib/schema/generate.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ const createSchema = (resources, lookups, additionalProperties) => {
type: 'array'
}
},
additionalProperties: false
additionalProperties
}
],
definitions: definitions
Expand Down
161 changes: 78 additions & 83 deletions lib/schema/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,31 @@ const processFiles = async ({ inputPath, fileContentsMap }) => {
const fileName = path.basename(inputPath);
const contents = await processFile({ filePath: inputPath });
if (fileName.endsWith('.zip')) {
const parsedContents = {};
Object.keys(contents).forEach(file => {
contents[file] = JSON.parse(contents[file]);
parsedContents[file] = JSON.parse(contents[file]);
});
Object.assign(fileContentsMap, contents);
Object.assign(fileContentsMap, parsedContents);
} else {
fileContentsMap[fileName] = contents;
fileContentsMap[inputPath] = contents;
}
} else if (stats.isDirectory()) {
const files = await fsPromises.readdir(inputPath);
for (const file of files) {
const fileName = path.basename(file);
if (fileName === '.DS_Store') continue;
const filePath = path.join(inputPath, file);
const content = await processFile({
filePath: path.join(inputPath, file)
filePath
});
if (fileName.endsWith('.zip')) {
const parsedContents = {};
Object.keys(content).forEach(file => {
content[file] = JSON.parse(content[file]);
parsedContents[file] = JSON.parse(content[file]);
});
Object.assign(fileContentsMap, content);
Object.assign(fileContentsMap, parsedContents);
} else {
fileContentsMap[fileName] = content;
fileContentsMap[filePath] = content;
}
}
} else {
Expand Down Expand Up @@ -114,46 +118,54 @@ const writeFile = async (path, data) => {
};

/**
* Here we try to parse out the field/resource/expansion for which the validation failed.
*
* Some examples of what ajv's error path `arr` might look like:
* - ["value","0","AboveGradeFinishedAreaSource"]
* - ["AboveGradeFinishedAreaSource"]
* - ["Media","0","MediaCategory"]
* - []
* - ["value","0","ListAgent","MemberAlternateId"]
*
* @param {Object} obj
* @param {string[]} obj.arr
* @param {boolean} obj.isValueArray
* @returns
* @param {Object} obj.metadataMap
* @param {string} obj.parentResourceName
* @returns {{parentResourceName: string, sourceModel: string?, sourceModelField: string?, fieldName: string?}}
*/
const parseNestedPropertyForResourceAndField = ({ arr, isValueArray }) => {
if (isValueArray) {
let arrIndexCount = 0;
arr.forEach(a => {
if (typeof Number(a) === 'number' && !isNaN(Number(a))) arrIndexCount++;
});
if (arrIndexCount >= 2) {
const lastElement = arr.at(-1);
if (typeof Number(lastElement) === 'number' && !isNaN(Number(lastElement))) {
return {
fieldName: arr[2]
};
const parseNestedPropertyForResourceAndField = ({ arr, metadataMap, parentResourceName }) => {
return arr.reduce(
(acc, field) => {
if (isNaN(Number(field)) && field !== 'value') {
if (field === parentResourceName) {
return {
...acc,
parentResourceName: field
};
} else if (metadataMap?.[parentResourceName]?.[field]?.isExpansion) {
return {
...acc,
sourceModel: metadataMap?.[parentResourceName]?.[field]?.typeName,
fieldName: field
};
} else {
if (acc?.sourceModel) {
return {
...acc,
sourceModelField: field
};
}
return {
...acc,
fieldName: field
};
}
} else {
return acc;
}
return {
resourceName: arr[2],
fieldName: arr[4]
};
} else {
return {
fieldName: arr[2]
};
}
} else {
if (arr.length === 3) {
return {
resourceName: arr[0],
fieldName: arr[2]
};
} else {
return {
fieldName: arr[0]
};
}
}
},
{ parentResourceName, sourceModel: null, sourceModelField: null, fieldName: null }
);
};

/**
Expand Down Expand Up @@ -215,27 +227,20 @@ const addCustomValidationForEnum = ajv => {
const version = validationContext.getVersion();
const validationConfig = validationContext.getValidationConfig();
const activeSchema = validationContext.getSchema();
const isValueArray = validationContext.getPayloadType() === 'MULTI';

const { fieldName } = parseNestedPropertyForResourceAndField({
const { fieldName, sourceModel, sourceModelField } = parseNestedPropertyForResourceAndField({
arr: nestedPayloadProperties,
isValueArray
metadataMap: activeSchema?.definitions?.MetadataMap ?? {},
parentResourceName: resourceName
});

const { isExpansion, typeName } = activeSchema?.definitions?.MetadataMap?.[resourceName]?.[fieldName] ?? {};

const expandedResource = isExpansion && typeName;

// eslint-disable-next-line prefer-destructuring
const expandedFieldName = isExpansion ? nestedPayloadProperties[3] : null; // this is how ajv orders it's erros. TODO: find a better was to do this.

if (typeof data === 'string') {
// Process the string to convert it into an array based on the enum definitions.
const { parsedEnumValues, isFlags } = processEnumerations({
lookupValue: data,
enums: schema,
resourceName: expandedResource || resourceName,
fieldName: expandedFieldName || fieldName,
resourceName: sourceModelField || resourceName,
fieldName: sourceModelField || fieldName,
schema: activeSchema
});

Expand All @@ -258,9 +263,11 @@ const addCustomValidationForEnum = ajv => {
} else {
// Collect the needed error data if validation fails
valid.push(false);
if (validationConfig?.[version]?.[resourceName]?.[fieldName]?.[IGNORE_ENUMS]) {
if (validationConfig?.[version]?.[sourceModel || resourceName]?.[sourceModelField || fieldName]?.[IGNORE_ENUMS]) {
// convert to warning is the failed field is ingored in the validation config
errorMessage.message = `The following enumerations in the ${fieldName} Field were not advertised. This will fail in Data Dictionary 2.1`;
errorMessage.message = `The following enumerations in the ${
sourceModelField || fieldName
} Field were not advertised. This will fail in Data Dictionary 2.1`;
errorMessage.isWarning = true;
}
validate.errors = validate.errors ?? [];
Expand Down Expand Up @@ -348,7 +355,7 @@ const addPayloadError = (resource, fileName, message, payloadErrors, stats) => {
* @param {string} field
*/
const normalizeAtField = (field = '') => {
if (field?.startsWith('@')) return field;
if (field?.startsWith('@')) return field.slice(1);
if (!field?.includes('@')) return field;

return field?.split('@')[0];
Expand Down Expand Up @@ -383,16 +390,19 @@ const combineErrors = ({
Object.entries(messages).forEach(([, files]) => {
if (!files) return;

const occurrences = Object.entries(files).flatMap(([fileName, values]) =>
Object.entries(values).map(([lookupValue, details]) => ({
const occurrences = Object.entries(files).flatMap(([fileName, values]) => {
const file = path.basename(fileName || '');
const pathName = fileName;
return Object.entries(values).map(([lookupValue, details]) => ({
count: details.occurrences,
...(fileName && { fileName }),
...(file && { fileName: file }),
...(pathName && { pathName }),
...(lookupValue && { lookupValue }),
...(details?.sourceModel && { sourceModel: details.sourceModel }),
...(details?.sourceModelField && { sourceModelField: details.sourceModelField }),
message: details.message
}))
);
}));
});

const message = occurrences?.[0]?.message ?? '';
addReport(
Expand Down Expand Up @@ -444,20 +454,6 @@ const combineErrors = ({
};
};

const parseSchemaPath = (path = '') => {
const schemaPathParts = path.split('/');
if (schemaPathParts?.[1] === 'definitions') {
return schemaPathParts[2];
}
return null;
};

const checkMapParams = (paramName = '') => {
if (paramName === '__proto__' || paramName === 'constructor' || paramName === 'prototype') {
throw new Error(`Invalid property value: '${paramName}'`);
}
};

/**
*
* @param {Object} obj
Expand Down Expand Up @@ -486,6 +482,7 @@ const updateCacheAndStats = ({
sourceModel,
sourceModelField
}) => {
const filePath = fileName;
if (!cache[resourceName]) {
cache[resourceName] = {};
}
Expand All @@ -498,12 +495,12 @@ const updateCacheAndStats = ({
cache[resourceName][failedItemName][message] = {};
}

if (!cache[resourceName][failedItemName][message][fileName]) {
cache[resourceName][failedItemName][message][fileName] = {};
if (!cache[resourceName][failedItemName][message][filePath]) {
cache[resourceName][failedItemName][message][filePath] = {};
}

if (!cache[resourceName][failedItemName][message][fileName][failedItemValue]) {
cache[resourceName][failedItemName][message][fileName][failedItemValue] = {
if (!cache[resourceName][failedItemName][message][filePath][failedItemValue]) {
cache[resourceName][failedItemName][message][filePath][failedItemValue] = {
// Capitalize first word like MUST, SHOULD, etc.
message: !message.startsWith(isWarning ? 'The' : 'Fields')
? message.slice(0, message.indexOf(' ')).toUpperCase() + message.slice(message.indexOf(' '), message.length)
Expand All @@ -513,7 +510,7 @@ const updateCacheAndStats = ({
sourceModelField
};
} else {
cache[resourceName][failedItemName][message][fileName][failedItemValue].occurrences++;
cache[resourceName][failedItemName][message][filePath][failedItemValue].occurrences++;
}

if (isWarning) {
Expand Down Expand Up @@ -576,8 +573,6 @@ module.exports = {
getValidDataDictionaryVersions,
isValidDataDictionaryVersion,
combineErrors,
parseSchemaPath,
checkMapParams,
updateCacheAndStats,
getResourceAndVersion,
getMaxLengthMessage
Expand Down
Loading

0 comments on commit 37531ad

Please sign in to comment.