Add JSON schemas documenting the v5.4 table- and field-level metadata

.github/workflows/validate-metadata.yml

@@ -0,0 +1,78 @@
+name: Validate metadata schemas
+
+on:
+  push:
+    paths:
+      - "inst/csv/OMOP_CDMv5.4_Table_Level.csv"
+      - "inst/csv/OMOP_CDMv5.4_Field_Level.csv"
+      - "inst/schema/**"
+      - ".github/workflows/validate-metadata.yml"
+  pull_request:
+    paths:
+      - "inst/csv/OMOP_CDMv5.4_Table_Level.csv"
+      - "inst/csv/OMOP_CDMv5.4_Field_Level.csv"
+      - "inst/schema/**"
+      - ".github/workflows/validate-metadata.yml"
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install jsonschema
+        run: pip install jsonschema
+
+      - name: Validate metadata CSVs against JSON Schemas
+        run: |
+          python - <<'EOF'
+          import csv, json, sys
+          from jsonschema import Draft4Validator
+
+          # CSV line numbers (1-based, header = line 1) with known formatting
+          # issues tracked in separate issues/PRs. Reported as warnings, not
+          # failures, so unrelated changes are not blocked. Remove entries as
+          # the fixes merge.
+          KNOWN_ISSUES = {
+              # Malformed cdm_source row: unquoted comma. See #744/#775, fix in #792.
+              ("inst/csv/OMOP_CDMv5.4_Field_Level.csv", 355),
+          }
+
+          FILES = [
+              ("inst/csv/OMOP_CDMv5.4_Table_Level.csv",
+               "inst/schema/OMOP_CDMv5.4_Table_Level.schema.json"),
+              ("inst/csv/OMOP_CDMv5.4_Field_Level.csv",
+               "inst/schema/OMOP_CDMv5.4_Field_Level.schema.json"),
+          ]
+
+          failures = warnings = 0
+          for csv_path, schema_path in FILES:
+              with open(schema_path) as f:
+                  validator = Draft4Validator(json.load(f))
+              with open(csv_path, newline="", encoding="utf-8-sig") as f:
+                  reader = csv.DictReader(f)
+                  for line, row in enumerate(reader, start=2):
+                      # A row with more cells than headers gets a None key from
+                      # DictReader; surface it as a structural problem.
+                      extra = row.pop(None, None)
+                      errors = [e.message for e in validator.iter_errors(row)]
+                      if extra is not None:
+                          errors.insert(0, f"row has {len(reader.fieldnames) + len(extra)} "
+                                           f"cells but header has {len(reader.fieldnames)}")
+                      if not errors:
+                          continue
+                      known = (csv_path, line) in KNOWN_ISSUES
+                      label = "warning (known issue)" if known else "FAILURE"
+                      warnings += known
+                      failures += not known
+                      print(f"{label}: {csv_path} line {line}")
+                      for e in errors:
+                          print(f"    {e}")
+
+          print(f"\n{failures} failure(s), {warnings} known-issue warning(s)")
+          sys.exit(1 if failures else 0)
+          EOF

extras/validateMetadataSchemas.R

@@ -0,0 +1,62 @@
+# Validate the OMOP CDM v5.4 metadata CSV files against the JSON Schemas in
+# inst/schema/. See inst/schema/README.md for the CSV-to-JSON representation
+# the schemas describe.
+#
+# This is a standalone maintenance script; it is not part of the package and
+# is not run by R CMD check. Run it from the repository root:
+#
+#   Rscript extras/validateMetadataSchemas.R
+#
+# Requires: readr, jsonlite, jsonvalidate (install from CRAN).
+
+library(readr)
+library(jsonlite)
+library(jsonvalidate)
+
+validateMetadataFile <- function(csvPath, schemaPath) {
+  cat(sprintf("\nValidating %s\n  against %s\n", csvPath, schemaPath))
+
+  # Read every cell as a character string and keep literal "NA" values as the
+  # string "NA" (na = character()), matching the representation documented in
+  # inst/schema/README.md.
+  metadata <- read_csv(
+    csvPath,
+    col_types = cols(.default = col_character()),
+    na = character(),
+    show_col_types = FALSE
+  )
+
+  validator <- json_validator(schemaPath, engine = "ajv")
+
+  failures <- 0
+  for (rowNumber in seq_len(nrow(metadata))) {
+    record <- as.list(metadata[rowNumber, , drop = FALSE])
+    recordJson <- toJSON(record, auto_unbox = TRUE, na = "string")
+    result <- validator(recordJson, verbose = TRUE)
+    if (!isTRUE(result)) {
+      failures <- failures + 1
+      # CSV line number = data row + 1 header line
+      cat(sprintf("  FAIL line %d (%s):\n", rowNumber + 1,
+                  paste(record$cdmTableName, record$cdmFieldName, sep = ".")))
+      print(attr(result, "errors"))
+    }
+  }
+
+  cat(sprintf("  %d rows checked, %d failures\n", nrow(metadata), failures))
+  failures
+}
+
+totalFailures <-
+  validateMetadataFile(
+    "inst/csv/OMOP_CDMv5.4_Table_Level.csv",
+    "inst/schema/OMOP_CDMv5.4_Table_Level.schema.json"
+  ) +
+  validateMetadataFile(
+    "inst/csv/OMOP_CDMv5.4_Field_Level.csv",
+    "inst/schema/OMOP_CDMv5.4_Field_Level.schema.json"
+  )
+
+if (totalFailures > 0) {
+  stop(sprintf("%d metadata rows failed schema validation", totalFailures))
+}
+cat("\nAll metadata rows conform to the schemas.\n")

inst/schema/OMOP_CDMv5.4_Field_Level.schema.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "https://ohdsi.github.io/CommonDataModel/schema/OMOP_CDMv5.4_Field_Level.schema.json",
+  "title": "OMOP CDM v5.4 Field-Level Metadata Row",
+  "description": "Describes one row of inst/csv/OMOP_CDMv5.4_Field_Level.csv represented as a JSON object. Property names match the CSV headers exactly (including 'unique DQ identifiers', which contains spaces), all values are strings, and the literal CSV value NA is preserved as the string \"NA\". The file is the ground truth for DDL generation (isRequired, cdmDatatype, isPrimaryKey, isForeignKey, fkTableName, fkFieldName) and the published CDM documentation (userGuidance, etlConventions); the fkDomain and fkClass columns correspond to the OHDSI DataQualityDashboard field-level checks of the same names.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "cdmTableName",
+    "cdmFieldName",
+    "isRequired",
+    "cdmDatatype",
+    "userGuidance",
+    "etlConventions",
+    "isPrimaryKey",
+    "isForeignKey",
+    "fkTableName",
+    "fkFieldName",
+    "fkDomain",
+    "fkClass",
+    "unique DQ identifiers"
+  ],
+  "properties": {
+    "cdmTableName": {
+      "description": "Name of the OMOP CDM table containing the field, in lowercase (e.g. person, visit_occurrence).",
+      "type": "string",
+      "minLength": 1
+    },
+    "cdmFieldName": {
+      "description": "Name of the field described by this row, in lowercase. Reserved words keep literal double quotes as part of the value (e.g. \"offset\" in the note_nlp table).",
+      "type": "string",
+      "minLength": 1
+    },
+    "isRequired": {
+      "description": "Whether the field must be populated. Yes results in a NOT NULL constraint in the generated DDL.",
+      "type": "string",
+      "enum": ["Yes", "No"]
+    },
+    "cdmDatatype": {
+      "description": "Database-neutral datatype used as input to DDL generation (e.g. integer, bigint, float, date, datetime, varchar(50), varchar(MAX)). Rendered to dialect-specific types by SqlRender.",
+      "type": "string",
+      "minLength": 1
+    },
+    "userGuidance": {
+      "description": "Guidance for users interpreting or querying the field, or the literal string NA when none is supplied. Used to generate the published CDM documentation.",
+      "type": "string",
+      "minLength": 1
+    },
+    "etlConventions": {
+      "description": "Conventions for populating the field during ETL, or the literal string NA when none is supplied. Used to generate the published CDM documentation.",
+      "type": "string",
+      "minLength": 1
+    },
+    "isPrimaryKey": {
+      "description": "Whether the field is (part of) the table's primary key. Used for primary-key generation in the DDL.",
+      "type": "string",
+      "enum": ["Yes", "No"]
+    },
+    "isForeignKey": {
+      "description": "Whether the field references a field in another CDM table. Used for foreign-key generation in the DDL.",
+      "type": "string",
+      "enum": ["Yes", "No"]
+    },
+    "fkTableName": {
+      "description": "Name of the referenced table, in uppercase (e.g. CONCEPT, PERSON), when isForeignKey is Yes; the literal string NA otherwise.",
+      "type": "string",
+      "minLength": 1
+    },
+    "fkFieldName": {
+      "description": "Name of the referenced field, in uppercase (e.g. CONCEPT_ID, PERSON_ID), when isForeignKey is Yes; the literal string NA otherwise.",
+      "type": "string",
+      "minLength": 1
+    },
+    "fkDomain": {
+      "description": "For standard concept ID fields: the vocabulary domain that concepts in this field must conform to (e.g. Gender for person.gender_concept_id); the literal string NA when no domain restriction applies. Input to the DataQualityDashboard fkDomain check. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkDomain.html",
+      "type": "string",
+      "minLength": 1
+    },
+    "fkClass": {
+      "description": "For standard concept ID fields: the concept class that concepts in this field must conform to, beyond the domain restriction (e.g. Ingredient for drug_era.drug_concept_id); the literal string NA when no class restriction applies. Input to the DataQualityDashboard fkClass check. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkClass.html",
+      "type": "string",
+      "minLength": 1
+    },
+    "unique DQ identifiers": {
+      "description": "Reserved column from the DataQualityDashboard threshold-file layout. Currently the literal string NA for every field in v5.4; the exact semantics should be confirmed with the CDM maintainers.",
+      "type": "string",
+      "minLength": 1
+    }
+  }
+}

inst/schema/OMOP_CDMv5.4_Table_Level.schema.json

@@ -0,0 +1,72 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "id": "https://ohdsi.github.io/CommonDataModel/schema/OMOP_CDMv5.4_Table_Level.schema.json",
+  "title": "OMOP CDM v5.4 Table-Level Metadata Row",
+  "description": "Describes one row of inst/csv/OMOP_CDMv5.4_Table_Level.csv represented as a JSON object. Property names match the CSV headers exactly, all values are strings, and the literal CSV value NA is preserved as the string \"NA\". The file follows the layout of the OHDSI DataQualityDashboard (DQD) table-level threshold files: a data-quality check column (e.g. measurePersonCompleteness) is paired with a companion <checkName>Threshold column.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "cdmTableName",
+    "schema",
+    "isRequired",
+    "conceptPrefix",
+    "measurePersonCompleteness",
+    "measurePersonCompletenessThreshold",
+    "validation",
+    "tableDescription",
+    "userGuidance",
+    "etlConventions"
+  ],
+  "properties": {
+    "cdmTableName": {
+      "description": "Name of the OMOP CDM table described by this row, in lowercase (e.g. person, visit_occurrence).",
+      "type": "string",
+      "minLength": 1
+    },
+    "schema": {
+      "description": "Logical schema group the table belongs to: CDM (clinical event and derived tables), VOCAB (standardized vocabulary tables), or RESULTS (cohort tables).",
+      "type": "string",
+      "enum": ["CDM", "VOCAB", "RESULTS"]
+    },
+    "isRequired": {
+      "description": "Whether the table is required for a conformant OMOP CDM instance.",
+      "type": "string",
+      "enum": ["Yes", "No"]
+    },
+    "conceptPrefix": {
+      "description": "Field-name prefix of the table's concept fields, set only for the clinical event tables (e.g. CONDITION_ for condition_occurrence, whose fields include condition_concept_id and condition_source_concept_id); the literal string NA otherwise. Historically part of the DataQualityDashboard threshold-file scaffolding; the exact downstream semantics should be confirmed with the CDM maintainers.",
+      "type": "string",
+      "minLength": 1
+    },
+    "measurePersonCompleteness": {
+      "description": "Whether the DataQualityDashboard measurePersonCompleteness check runs for this table. The check measures the number and percent of persons in the CDM that do not have at least one record in this table. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/measurePersonCompleteness.html",
+      "type": "string",
+      "enum": ["Yes", "No"]
+    },
+    "measurePersonCompletenessThreshold": {
+      "description": "Failure threshold (percent) for the measurePersonCompleteness check, as a numeric string (e.g. 0); the literal string NA when the check does not apply. Follows the DataQualityDashboard convention that a check column named <checkName> is paired with a <checkName>Threshold column.",
+      "type": "string",
+      "minLength": 1
+    },
+    "validation": {
+      "description": "Reserved column from the DataQualityDashboard threshold-file layout, intended for validation-context data-quality checks. Currently the literal string NA for every table in v5.4; the exact semantics should be confirmed with the CDM maintainers.",
+      "type": "string",
+      "minLength": 1
+    },
+    "tableDescription": {
+      "description": "Description of the table and its role in the OMOP CDM. Used to generate the published CDM documentation.",
+      "type": "string",
+      "minLength": 1
+    },
+    "userGuidance": {
+      "description": "Guidance for users interpreting or querying data in this table, or the literal string NA when none is supplied. Used to generate the published CDM documentation.",
+      "type": "string",
+      "minLength": 1
+    },
+    "etlConventions": {
+      "description": "Conventions for populating this table during ETL, or the literal string NA when none is supplied. Used to generate the published CDM documentation.",
+      "type": "string",
+      "minLength": 1
+    }
+  }
+}