diff --git a/.github/workflows/validate-metadata.yml b/.github/workflows/validate-metadata.yml new file mode 100644 index 00000000..b771ec47 --- /dev/null +++ b/.github/workflows/validate-metadata.yml @@ -0,0 +1,78 @@ +name: Validate metadata schemas + +on: + push: + paths: + - "inst/csv/OMOP_CDMv5.4_Table_Level.csv" + - "inst/csv/OMOP_CDMv5.4_Field_Level.csv" + - "inst/schema/**" + - ".github/workflows/validate-metadata.yml" + pull_request: + paths: + - "inst/csv/OMOP_CDMv5.4_Table_Level.csv" + - "inst/csv/OMOP_CDMv5.4_Field_Level.csv" + - "inst/schema/**" + - ".github/workflows/validate-metadata.yml" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install jsonschema + run: pip install jsonschema + + - name: Validate metadata CSVs against JSON Schemas + run: | + python - <<'EOF' + import csv, json, sys + from jsonschema import Draft4Validator + + # CSV line numbers (1-based, header = line 1) with known formatting + # issues tracked in separate issues/PRs. Reported as warnings, not + # failures, so unrelated changes are not blocked. Remove entries as + # the fixes merge. + KNOWN_ISSUES = { + # Malformed cdm_source row: unquoted comma. See #744/#775, fix in #792. + ("inst/csv/OMOP_CDMv5.4_Field_Level.csv", 355), + } + + FILES = [ + ("inst/csv/OMOP_CDMv5.4_Table_Level.csv", + "inst/schema/OMOP_CDMv5.4_Table_Level.schema.json"), + ("inst/csv/OMOP_CDMv5.4_Field_Level.csv", + "inst/schema/OMOP_CDMv5.4_Field_Level.schema.json"), + ] + + failures = warnings = 0 + for csv_path, schema_path in FILES: + with open(schema_path) as f: + validator = Draft4Validator(json.load(f)) + with open(csv_path, newline="", encoding="utf-8-sig") as f: + reader = csv.DictReader(f) + for line, row in enumerate(reader, start=2): + # A row with more cells than headers gets a None key from + # DictReader; surface it as a structural problem. + extra = row.pop(None, None) + errors = [e.message for e in validator.iter_errors(row)] + if extra is not None: + errors.insert(0, f"row has {len(reader.fieldnames) + len(extra)} " + f"cells but header has {len(reader.fieldnames)}") + if not errors: + continue + known = (csv_path, line) in KNOWN_ISSUES + label = "warning (known issue)" if known else "FAILURE" + warnings += known + failures += not known + print(f"{label}: {csv_path} line {line}") + for e in errors: + print(f" {e}") + + print(f"\n{failures} failure(s), {warnings} known-issue warning(s)") + sys.exit(1 if failures else 0) + EOF diff --git a/extras/validateMetadataSchemas.R b/extras/validateMetadataSchemas.R new file mode 100644 index 00000000..c7439281 --- /dev/null +++ b/extras/validateMetadataSchemas.R @@ -0,0 +1,62 @@ +# Validate the OMOP CDM v5.4 metadata CSV files against the JSON Schemas in +# inst/schema/. See inst/schema/README.md for the CSV-to-JSON representation +# the schemas describe. +# +# This is a standalone maintenance script; it is not part of the package and +# is not run by R CMD check. Run it from the repository root: +# +# Rscript extras/validateMetadataSchemas.R +# +# Requires: readr, jsonlite, jsonvalidate (install from CRAN). + +library(readr) +library(jsonlite) +library(jsonvalidate) + +validateMetadataFile <- function(csvPath, schemaPath) { + cat(sprintf("\nValidating %s\n against %s\n", csvPath, schemaPath)) + + # Read every cell as a character string and keep literal "NA" values as the + # string "NA" (na = character()), matching the representation documented in + # inst/schema/README.md. + metadata <- read_csv( + csvPath, + col_types = cols(.default = col_character()), + na = character(), + show_col_types = FALSE + ) + + validator <- json_validator(schemaPath, engine = "ajv") + + failures <- 0 + for (rowNumber in seq_len(nrow(metadata))) { + record <- as.list(metadata[rowNumber, , drop = FALSE]) + recordJson <- toJSON(record, auto_unbox = TRUE, na = "string") + result <- validator(recordJson, verbose = TRUE) + if (!isTRUE(result)) { + failures <- failures + 1 + # CSV line number = data row + 1 header line + cat(sprintf(" FAIL line %d (%s):\n", rowNumber + 1, + paste(record$cdmTableName, record$cdmFieldName, sep = "."))) + print(attr(result, "errors")) + } + } + + cat(sprintf(" %d rows checked, %d failures\n", nrow(metadata), failures)) + failures +} + +totalFailures <- + validateMetadataFile( + "inst/csv/OMOP_CDMv5.4_Table_Level.csv", + "inst/schema/OMOP_CDMv5.4_Table_Level.schema.json" + ) + + validateMetadataFile( + "inst/csv/OMOP_CDMv5.4_Field_Level.csv", + "inst/schema/OMOP_CDMv5.4_Field_Level.schema.json" + ) + +if (totalFailures > 0) { + stop(sprintf("%d metadata rows failed schema validation", totalFailures)) +} +cat("\nAll metadata rows conform to the schemas.\n") diff --git a/inst/schema/OMOP_CDMv5.4_Field_Level.schema.json b/inst/schema/OMOP_CDMv5.4_Field_Level.schema.json new file mode 100644 index 00000000..2f13044a --- /dev/null +++ b/inst/schema/OMOP_CDMv5.4_Field_Level.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://ohdsi.github.io/CommonDataModel/schema/OMOP_CDMv5.4_Field_Level.schema.json", + "title": "OMOP CDM v5.4 Field-Level Metadata Row", + "description": "Describes one row of inst/csv/OMOP_CDMv5.4_Field_Level.csv represented as a JSON object. Property names match the CSV headers exactly (including 'unique DQ identifiers', which contains spaces), all values are strings, and the literal CSV value NA is preserved as the string \"NA\". The file is the ground truth for DDL generation (isRequired, cdmDatatype, isPrimaryKey, isForeignKey, fkTableName, fkFieldName) and the published CDM documentation (userGuidance, etlConventions); the fkDomain and fkClass columns correspond to the OHDSI DataQualityDashboard field-level checks of the same names.", + "type": "object", + "additionalProperties": false, + "required": [ + "cdmTableName", + "cdmFieldName", + "isRequired", + "cdmDatatype", + "userGuidance", + "etlConventions", + "isPrimaryKey", + "isForeignKey", + "fkTableName", + "fkFieldName", + "fkDomain", + "fkClass", + "unique DQ identifiers" + ], + "properties": { + "cdmTableName": { + "description": "Name of the OMOP CDM table containing the field, in lowercase (e.g. person, visit_occurrence).", + "type": "string", + "minLength": 1 + }, + "cdmFieldName": { + "description": "Name of the field described by this row, in lowercase. Reserved words keep literal double quotes as part of the value (e.g. \"offset\" in the note_nlp table).", + "type": "string", + "minLength": 1 + }, + "isRequired": { + "description": "Whether the field must be populated. Yes results in a NOT NULL constraint in the generated DDL.", + "type": "string", + "enum": ["Yes", "No"] + }, + "cdmDatatype": { + "description": "Database-neutral datatype used as input to DDL generation (e.g. integer, bigint, float, date, datetime, varchar(50), varchar(MAX)). Rendered to dialect-specific types by SqlRender.", + "type": "string", + "minLength": 1 + }, + "userGuidance": { + "description": "Guidance for users interpreting or querying the field, or the literal string NA when none is supplied. Used to generate the published CDM documentation.", + "type": "string", + "minLength": 1 + }, + "etlConventions": { + "description": "Conventions for populating the field during ETL, or the literal string NA when none is supplied. Used to generate the published CDM documentation.", + "type": "string", + "minLength": 1 + }, + "isPrimaryKey": { + "description": "Whether the field is (part of) the table's primary key. Used for primary-key generation in the DDL.", + "type": "string", + "enum": ["Yes", "No"] + }, + "isForeignKey": { + "description": "Whether the field references a field in another CDM table. Used for foreign-key generation in the DDL.", + "type": "string", + "enum": ["Yes", "No"] + }, + "fkTableName": { + "description": "Name of the referenced table, in uppercase (e.g. CONCEPT, PERSON), when isForeignKey is Yes; the literal string NA otherwise.", + "type": "string", + "minLength": 1 + }, + "fkFieldName": { + "description": "Name of the referenced field, in uppercase (e.g. CONCEPT_ID, PERSON_ID), when isForeignKey is Yes; the literal string NA otherwise.", + "type": "string", + "minLength": 1 + }, + "fkDomain": { + "description": "For standard concept ID fields: the vocabulary domain that concepts in this field must conform to (e.g. Gender for person.gender_concept_id); the literal string NA when no domain restriction applies. Input to the DataQualityDashboard fkDomain check. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkDomain.html", + "type": "string", + "minLength": 1 + }, + "fkClass": { + "description": "For standard concept ID fields: the concept class that concepts in this field must conform to, beyond the domain restriction (e.g. Ingredient for drug_era.drug_concept_id); the literal string NA when no class restriction applies. Input to the DataQualityDashboard fkClass check. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkClass.html", + "type": "string", + "minLength": 1 + }, + "unique DQ identifiers": { + "description": "Reserved column from the DataQualityDashboard threshold-file layout. Currently the literal string NA for every field in v5.4; the exact semantics should be confirmed with the CDM maintainers.", + "type": "string", + "minLength": 1 + } + } +} diff --git a/inst/schema/OMOP_CDMv5.4_Table_Level.schema.json b/inst/schema/OMOP_CDMv5.4_Table_Level.schema.json new file mode 100644 index 00000000..c2498396 --- /dev/null +++ b/inst/schema/OMOP_CDMv5.4_Table_Level.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://ohdsi.github.io/CommonDataModel/schema/OMOP_CDMv5.4_Table_Level.schema.json", + "title": "OMOP CDM v5.4 Table-Level Metadata Row", + "description": "Describes one row of inst/csv/OMOP_CDMv5.4_Table_Level.csv represented as a JSON object. Property names match the CSV headers exactly, all values are strings, and the literal CSV value NA is preserved as the string \"NA\". The file follows the layout of the OHDSI DataQualityDashboard (DQD) table-level threshold files: a data-quality check column (e.g. measurePersonCompleteness) is paired with a companion Threshold column.", + "type": "object", + "additionalProperties": false, + "required": [ + "cdmTableName", + "schema", + "isRequired", + "conceptPrefix", + "measurePersonCompleteness", + "measurePersonCompletenessThreshold", + "validation", + "tableDescription", + "userGuidance", + "etlConventions" + ], + "properties": { + "cdmTableName": { + "description": "Name of the OMOP CDM table described by this row, in lowercase (e.g. person, visit_occurrence).", + "type": "string", + "minLength": 1 + }, + "schema": { + "description": "Logical schema group the table belongs to: CDM (clinical event and derived tables), VOCAB (standardized vocabulary tables), or RESULTS (cohort tables).", + "type": "string", + "enum": ["CDM", "VOCAB", "RESULTS"] + }, + "isRequired": { + "description": "Whether the table is required for a conformant OMOP CDM instance.", + "type": "string", + "enum": ["Yes", "No"] + }, + "conceptPrefix": { + "description": "Field-name prefix of the table's concept fields, set only for the clinical event tables (e.g. CONDITION_ for condition_occurrence, whose fields include condition_concept_id and condition_source_concept_id); the literal string NA otherwise. Historically part of the DataQualityDashboard threshold-file scaffolding; the exact downstream semantics should be confirmed with the CDM maintainers.", + "type": "string", + "minLength": 1 + }, + "measurePersonCompleteness": { + "description": "Whether the DataQualityDashboard measurePersonCompleteness check runs for this table. The check measures the number and percent of persons in the CDM that do not have at least one record in this table. See https://ohdsi.github.io/DataQualityDashboard/articles/checks/measurePersonCompleteness.html", + "type": "string", + "enum": ["Yes", "No"] + }, + "measurePersonCompletenessThreshold": { + "description": "Failure threshold (percent) for the measurePersonCompleteness check, as a numeric string (e.g. 0); the literal string NA when the check does not apply. Follows the DataQualityDashboard convention that a check column named is paired with a Threshold column.", + "type": "string", + "minLength": 1 + }, + "validation": { + "description": "Reserved column from the DataQualityDashboard threshold-file layout, intended for validation-context data-quality checks. Currently the literal string NA for every table in v5.4; the exact semantics should be confirmed with the CDM maintainers.", + "type": "string", + "minLength": 1 + }, + "tableDescription": { + "description": "Description of the table and its role in the OMOP CDM. Used to generate the published CDM documentation.", + "type": "string", + "minLength": 1 + }, + "userGuidance": { + "description": "Guidance for users interpreting or querying data in this table, or the literal string NA when none is supplied. Used to generate the published CDM documentation.", + "type": "string", + "minLength": 1 + }, + "etlConventions": { + "description": "Conventions for populating this table during ETL, or the literal string NA when none is supplied. Used to generate the published CDM documentation.", + "type": "string", + "minLength": 1 + } + } +} diff --git a/inst/schema/README.md b/inst/schema/README.md new file mode 100644 index 00000000..5129f1d0 --- /dev/null +++ b/inst/schema/README.md @@ -0,0 +1,110 @@ +# OMOP CDM Metadata Schemas + +This directory contains [JSON Schemas](https://json-schema.org/) (draft-04) that formally +describe the metadata columns of the OMOP CDM definition CSV files, as requested in +[discussion #746](https://github.com/OHDSI/CommonDataModel/discussions/746). + +| Schema | Describes one row of | +| --- | --- | +| `OMOP_CDMv5.4_Table_Level.schema.json` | `inst/csv/OMOP_CDMv5.4_Table_Level.csv` | +| `OMOP_CDMv5.4_Field_Level.schema.json` | `inst/csv/OMOP_CDMv5.4_Field_Level.csv` | + +These schemas document the **CDM definition metadata** — the rows that describe the +model itself and drive DDL generation, the published documentation, and the +[DataQualityDashboard](https://github.com/OHDSI/DataQualityDashboard) checks. They do +not describe or validate an instantiated patient-level database. + +## CSV-to-JSON representation + +JSON Schema validates JSON, not CSV. Each schema describes **one CSV row** converted +to a JSON object under the following rules: + +- JSON property names are identical to the CSV column headers, including + `unique DQ identifiers` (which contains spaces). +- Every CSV value is represented as a JSON string, even when it looks numeric + (e.g. `measurePersonCompletenessThreshold` of `0` becomes `"0"`). +- The literal CSV value `NA` is preserved as the JSON string `"NA"`; it is not + converted to JSON `null`. + +Example (from the table-level file): + +```json +{ + "cdmTableName": "person", + "schema": "CDM", + "isRequired": "Yes", + "conceptPrefix": "NA", + "measurePersonCompleteness": "No", + "measurePersonCompletenessThreshold": "NA", + "validation": "NA", + "tableDescription": "This table serves as the central identity management for all Persons in the database...", + "userGuidance": "All records in this table are independent Persons.", + "etlConventions": "All Persons in a database needs one record in this table..." +} +``` + +## Relationship to the DataQualityDashboard + +The CSV layout follows the DataQualityDashboard (DQD) threshold-file convention: +a data-quality check column named `` (with values `Yes`/`No`) is paired +with a `Threshold` column giving the check's failure threshold. In this +repository's copies, the table-level file carries the `measurePersonCompleteness` +check and the field-level file carries the `fkDomain` and `fkClass` check inputs; +DQD ships extended copies of these files with the full set of check columns. + +Authoritative definitions of the checks: + +- [measurePersonCompleteness](https://ohdsi.github.io/DataQualityDashboard/articles/checks/measurePersonCompleteness.html) +- [fkDomain](https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkDomain.html) +- [fkClass](https://ohdsi.github.io/DataQualityDashboard/articles/checks/fkClass.html) + +The `conceptPrefix`, `validation`, and `unique DQ identifiers` columns are part of +the historical threshold-file layout and are not currently consumed by the +CommonDataModel or DataQualityDashboard code; their descriptions in the schemas are +marked as pending confirmation by the CDM maintainers. + +## Observed cross-field invariants + +The following relationships hold for every well-formed row of the v5.4 files. They +are documented here as **observed invariants** rather than encoded as schema rules, +because they have not been confirmed as intentional conventions by the CDM +maintainers; a future revision may promote them into the schemas. + +Table level: + +- `measurePersonCompleteness` is `Yes` if and only if + `measurePersonCompletenessThreshold` is not `NA`. +- `conceptPrefix` is set (not `NA`) exactly for the nine clinical event tables + (`condition_occurrence`, `device_exposure`, `drug_exposure`, `measurement`, + `observation`, `procedure_occurrence`, `specimen`, `visit_detail`, + `visit_occurrence`), and lowercasing the prefix and appending `concept_id` + always yields a real field of that table (e.g. `CONDITION_` → + `condition_concept_id`). + +Field level: + +- `isForeignKey` is `Yes` if and only if both `fkTableName` and `fkFieldName` + are not `NA`. +- Whenever `fkDomain` or `fkClass` is set (not `NA`), the field is a foreign key + to the `CONCEPT` table. +- Whenever `fkClass` is set, `fkDomain` is also set. + +## Validating the CSVs against the schemas + +Two optional tools validate every CSV row against these schemas using the +representation rules above: + +- `extras/validateMetadataSchemas.R` — standalone R script (requires `readr`, + `jsonlite`, `jsonvalidate`); run from the repository root with + `Rscript extras/validateMetadataSchemas.R`. It is not part of the package and + is not run by `R CMD check`. +- `.github/workflows/validate-metadata.yml` — GitHub Actions workflow (Python) + that runs automatically when the v5.4 metadata CSVs or the schemas change. + +Known issue: one row of `OMOP_CDMv5.4_Field_Level.csv` (`cdm_source.` +`source_documentation_reference`, line 355) currently contains an unquoted comma +and fails validation; this is tracked in +[#744](https://github.com/OHDSI/CommonDataModel/issues/744) / +[#775](https://github.com/OHDSI/CommonDataModel/issues/775) and fixed by +[#792](https://github.com/OHDSI/CommonDataModel/pull/792). The CI workflow reports +it as a known-issue warning until that fix merges.