Merge pull request #12 from duckduckgo/la/global-ignore-and-json5-support

Support JSON5, rename common_* files, add global ignore list

Author: sammacbeth

global_pixel_definitions/ignore_params.json

@@ -0,0 +1,22 @@
+{
+    "kp": {
+        "key": "kp",
+        "description": "",
+        "enum": [1]
+    },
+    "p": {
+        "key": "p",
+        "description": "",
+        "enum": [1]
+    },
+    "test": {
+        "key": "test",
+        "description": "",
+        "enum": [1]
+    },
+    "scrlybrkr": {
+        "key": "scrlybrkr",
+        "description": "Added by 3rd party school proxy",
+        "type": "number"
+    }
+}

live_validation_scripts/validate_live_pixel.mjs

@@ -17,10 +17,13 @@ function main(mainDir, csvFile) {
     const productDef = fileUtils.readProductDef(mainDir);
     const commonParams = fileUtils.readCommonParams(mainDir);
     const commonSuffixes = fileUtils.readCommonSuffixes(mainDir);
+    const globalIgnoreParams = fileUtils.readIgnoreParams(fileUtils.GLOBAL_PIXEL_DIR);
 
     const tokenizedPixels = fileUtils.readTokenizedPixels(mainDir);
     const paramsValidator = new ParamsValidator(commonParams, commonSuffixes);
-    const ignoreParams = fileUtils.readIgnoreParams(mainDir);
+    const pixelIgnoreParams = fileUtils.readIgnoreParams(mainDir);
+
+    const ignoreParams = [...(Object.values(pixelIgnoreParams) || []), ...Object.values(globalIgnoreParams)];
 
     const liveValidator = new LivePixelsValidator(tokenizedPixels, productDef, ignoreParams, paramsValidator);
     let processedPixels = 0;

src/file_utils.mjs

@@ -7,15 +7,52 @@ import path from 'path';
 import JSON5 from 'json5';
 
 const RESULTS_DIR = 'pixel_processing_results';
+export const GLOBAL_PIXEL_DIR = 'global_pixel_definitions';
+
+/**
+ * Attempt to read and parse a file using JSON5. Tries .json
+ * first but will try to json5 if missing.
+ *
+ * @param {string} filePath - Absolute path to a file
+ * @returns {object} Parsed file content
+ * @throws Will throw an error if neither file exists.
+ */
+function parseFile(filePath) {
+    let resolvedPath = filePath;
+    if (!fs.existsSync(resolvedPath)) {
+        // Try the '.json5' fallback
+        const { dir, name } = path.parse(filePath);
+        const altPath = path.join(dir, `${name}.json5`);
+        if (fs.existsSync(altPath)) {
+            resolvedPath = altPath;
+        } else {
+            throw new Error(`Neither ${filePath} nor ${altPath} exist.`);
+        }
+    }
+    const fileContent = fs.readFileSync(resolvedPath, 'utf8');
+    return JSON5.parse(fileContent);
+}
+
+/**
+ * Builds a file path from the main pixel directory and the given filename,
+ * then parses the file.
+ *
+ * @param {string} mainPixelDir - path to the main pixels directory
+ * @param {string} filename - file name (with extension) to read
+ * @returns {object} Parsed file content
+ */
+function readSchemaFile(mainPixelDir, filename) {
+    const filePath = path.join(mainPixelDir, filename);
+    return parseFile(filePath);
+}
 
 /**
  * Read common parameters
  * @param {string} mainPixelDir - path to the main pixels directory
  * @returns {object} common parameters
  */
 export function readCommonParams(mainPixelDir) {
-    console.log(mainPixelDir);
-    return parseFile(path.join(mainPixelDir, 'common_params.json'));
+    return readSchemaFile(mainPixelDir, 'params_dictionary.json');
 }
 
 /**
@@ -24,7 +61,7 @@ export function readCommonParams(mainPixelDir) {
  * @returns {object} common suffixes
  */
 export function readCommonSuffixes(mainPixelDir) {
-    return parseFile(path.join(mainPixelDir, 'common_suffixes.json'));
+    return readSchemaFile(mainPixelDir, 'suffixes_dictionary.json');
 }
 
 /**
@@ -33,13 +70,13 @@ export function readCommonSuffixes(mainPixelDir) {
  * @returns {object} ignore parameters
  */
 export function readIgnoreParams(mainPixelDir) {
-    return parseFile(path.join(mainPixelDir, 'ignore_params.json'));
+    return readSchemaFile(mainPixelDir, 'ignore_params.json');
 }
 
 /**
- * Get product definition path
+ * Get product definition path based on mainPixelDir.
  * @param {string} mainPixelDir - path to the main pixels directory
- * @returns {string} product definition path
+ * @returns {string} product definition file path
  */
 export function getProductDefPath(mainPixelDir) {
     return path.join(mainPixelDir, 'product.json');
@@ -67,13 +104,24 @@ export function getResultsDir(mainPixelDir) {
     return resultsDir;
 }
 
+/**
+ * Get path to a file inside the results directory.
+ *
+ * @param {string} mainPixelDir - path to the main pixels directory
+ * @param {string} filename - file name within the results directory
+ * @returns {string} Absolute path to the file in results
+ */
+function getResultsFilePath(mainPixelDir, filename) {
+    return path.join(getResultsDir(mainPixelDir), filename);
+}
+
 /**
  * Get path to pixel errors encountered during live validation
  * @param {string} mainPixelDir - path to the main pixels directory
  * @returns {string} pixel errors path
  */
 export function getPixelErrorsPath(mainPixelDir) {
-    return path.join(getResultsDir(mainPixelDir), 'pixel_errors.json');
+    return getResultsFilePath(mainPixelDir, 'pixel_errors.json');
 }
 
 /**
@@ -82,16 +130,16 @@ export function getPixelErrorsPath(mainPixelDir) {
  * @returns {string} undocumented pixels path
  */
 export function getUndocumentedPixelsPath(mainPixelDir) {
-    return path.join(getResultsDir(mainPixelDir), 'undocumented_pixels.json');
+    return getResultsFilePath(mainPixelDir, 'undocumented_pixels.json');
 }
 
 /**
- * Get tokenized pixels path and creates the path if it doesn't exist
+ * Get tokenized pixels path
  * @param {string} mainPixelDir - path to the main pixels directory
  * @returns {string} tokenized pixels path
  */
 export function getTokenizedPixelsPath(mainPixelDir) {
-    return path.join(getResultsDir(mainPixelDir), 'tokenized_pixels.json');
+    return getResultsFilePath(mainPixelDir, 'tokenized_pixels.json');
 }
 
 /**
@@ -102,7 +150,3 @@ export function getTokenizedPixelsPath(mainPixelDir) {
 export function readTokenizedPixels(mainPixelDir) {
     return parseFile(getTokenizedPixelsPath(mainPixelDir));
 }
-
-function parseFile(filePath) {
-    return JSON5.parse(fs.readFileSync(filePath).toString());
-}

tests/test_data/invalid/ignore_params.json

@@ -1,17 +1 @@
-{
-    "kp": {
-        "key": "kp",
-        "description": "",
-        "enum": [1]
-    },
-    "p": {
-        "key": "p",
-        "description": "",
-        "enum": [1]
-    },
-    "test": {
-        "key": "test",
-        "description": "",
-        "enum": [1]
-    }
-}
+{}

tests/test_data/invalid/common_params.json renamed to tests/test_data/invalid/params_dictionary.json

@@ -1,17 +1 @@
-{
-    "kp": {
-        "key": "kp",
-        "description": "",
-        "enum": [1]
-    },
-    "p": {
-        "key": "p",
-        "description": "",
-        "enum": [1]
-    },
-    "test": {
-        "key": "test",
-        "description": "",
-        "enum": [1]
-    }
-}
+{}

tests/test_data/invalid/common_suffixes.json renamed to tests/test_data/invalid/suffixes_dictionary.json

@@ -1,17 +1 @@
-{
-    "kp": {
-        "key": "kp",
-        "description": "",
-        "enum": [1]
-    },
-    "p": {
-        "key": "p",
-        "description": "",
-        "enum": [1]
-    },
-    "test": {
-        "key": "test",
-        "description": "",
-        "enum": [1]
-    }
-}
+{}