Intro of automated and edited folders of metastore

stephengoldbaum · Jun 18, 2024 · 9846296 · 9846296
1 parent 80417b2
commit 9846296
Show file tree

Hide file tree

Showing 30 changed files with 104 additions and 85 deletions.
diff --git a/DataThread/README.md b/DataThread/README.md
@@ -40,17 +40,17 @@ An example can be found in the [metastore folder](metastore). Some things to loo
 * Run `npm install`
 
 #### People example
-  * `node src/request_processor.js` --baseDir example/metastore
+  * `node src/request_processor.js --baseDir example/metastore`
   * Open a browser to the written link.
   * Execute any of the [sample queries](example_graphql_queries.json).
 
 #### Clean example
 * Run the Request Process
-  * `node src/request_processor.js` --baseDir data
+  * `node src/main/js/request_processor.js --baseDir data`
   * `curl -X POST -H "Content-Type: application/json" -d @example/test.foo.element.json http://localhost:3000/element`
   * `curl -X POST -H "Content-Type: application/json" -d @example/test.foo.dataset.json http://localhost:3000/dataset`
 * Run Query Processor
-  * `node src/graphql_server.js` --baseDir data
+  * `node src/main/js/graphql_server.js` --baseDir data
   * Open a browser to the written link.
   * Execute any of the [sample queries](example_graphql_queries.json).
 

diff --git a/DataThread/Specification.md b/DataThread/Specification.md
@@ -39,8 +39,11 @@ The specification defines core Metadata Sets for Dataset and Element. All other
 Important components of the Specification are:
 ### Identifiers
 Every Dataset and Element must be uniquely identified. It is the responsibility of the owner to set the identification. To avoid clashes, the Specification follows the [Uniform Resource Identifier](https://en.wikipedia.org/wiki/Uniform_Resource_Identifier) standard with the following URN structure of:
+
   * `[type]:[/domain][/subdomain?]:[name]`.
 
+Note that domains and names can contain spaces, which allows the identifier to match the actual words.
+
 ### Core Data Structures
 #### Dataset Structure
 

diff --git a/...ample/metastore/core/integer.element.json → ...store/automated/core/integer.element.json b/...ample/metastore/core/integer.element.json → ...store/automated/core/integer.element.json
diff --git a/...d/example/metastore/core/nil.element.json → ...metastore/automated/core/nil.element.json b/...d/example/metastore/core/nil.element.json → ...metastore/automated/core/nil.element.json
diff --git a/...mple/metastore/person/active.element.json → ...tore/automated/person/active.element.json b/...mple/metastore/person/active.element.json → ...tore/automated/person/active.element.json
diff --git a/...example/metastore/person/age.element.json → ...tastore/automated/person/age.element.json b/...example/metastore/person/age.element.json → ...tastore/automated/person/age.element.json
diff --git a/...le/metastore/person/currency.element.json → ...re/automated/person/currency.element.json b/...le/metastore/person/currency.element.json → ...re/automated/person/currency.element.json
diff --git a/...tastore/person/date_of_birth.element.json → ...tomated/person/date_of_birth.element.json b/...tastore/person/date_of_birth.element.json → ...tomated/person/date_of_birth.element.json
diff --git a/.../metastore/person/first_name.element.json → .../automated/person/first_name.element.json b/.../metastore/person/first_name.element.json → .../automated/person/first_name.element.json
diff --git a/...mple/metastore/person/first_name.fdc.json → ...tore/automated/person/first_name.fdc.json b/...mple/metastore/person/first_name.fdc.json → ...tore/automated/person/first_name.fdc.json
diff --git a/.../example/metastore/person/id.element.json → ...etastore/automated/person/id.element.json b/.../example/metastore/person/id.element.json → ...etastore/automated/person/id.element.json
diff --git a/...e/metastore/person/last_name.element.json → ...e/automated/person/last_name.element.json b/...e/metastore/person/last_name.element.json → ...e/automated/person/last_name.element.json
diff --git a/...etastore/person/last_session.element.json → ...utomated/person/last_session.element.json b/...etastore/person/last_session.element.json → ...utomated/person/last_session.element.json
diff --git a/...mple/metastore/person/people.dataset.json → ...tore/automated/person/people.dataset.json b/...mple/metastore/person/people.dataset.json → ...tore/automated/person/people.dataset.json
diff --git a/...astore/person/savings_amount.element.json → ...omated/person/savings_amount.element.json b/...astore/person/savings_amount.element.json → ...omated/person/savings_amount.element.json
diff --git a/...rson/savings_amount_currency.element.json → ...rson/savings_amount_currency.element.json b/...rson/savings_amount_currency.element.json → ...rson/savings_amount_currency.element.json
diff --git a/...on/users#unknownElementField.element.json → ...on/users#unknownElementField.element.json b/...on/users#unknownElementField.element.json → ...on/users#unknownElementField.element.json
diff --git a/...ample/metastore/person/users.dataset.json → ...store/automated/person/users.dataset.json b/...ample/metastore/person/users.dataset.json → ...store/automated/person/users.dataset.json
diff --git a/...ample/metastore/person/zeta#id.field.json → ...store/automated/person/zeta#id.field.json b/...ample/metastore/person/zeta#id.field.json → ...store/automated/person/zeta#id.field.json
diff --git a/...son/zeta#unknownElementField.element.json → ...son/zeta#unknownElementField.element.json b/...son/zeta#unknownElementField.element.json → ...son/zeta#unknownElementField.element.json
diff --git a/...xample/metastore/person/zeta.dataset.json → ...astore/automated/person/zeta.dataset.json b/...xample/metastore/person/zeta.dataset.json → ...astore/automated/person/zeta.dataset.json
diff --git a/...le/metastore/person/age.element_info.json → ...store/edited/person/age.element_info.json b/...le/metastore/person/age.element_info.json → ...store/edited/person/age.element_info.json
diff --git a/...re/person/date_of_birth.element_info.json → ...ed/person/date_of_birth.element_info.json b/...re/person/date_of_birth.element_info.json → ...ed/person/date_of_birth.element_info.json
diff --git a/...store/person/first_name.element_info.json → ...dited/person/first_name.element_info.json b/...store/person/first_name.element_info.json → ...dited/person/first_name.element_info.json
diff --git a/...ple/metastore/person/id.element_info.json → ...astore/edited/person/id.element_info.json b/...ple/metastore/person/id.element_info.json → ...astore/edited/person/id.element_info.json
diff --git a/...astore/person/last_name.element.info.json → ...edited/person/last_name.element.info.json b/...astore/person/last_name.element.info.json → ...edited/person/last_name.element.info.json
diff --git a/...e/person/savings_amount.element_info.json → ...d/person/savings_amount.element_info.json b/...e/person/savings_amount.element_info.json → ...d/person/savings_amount.element_info.json
diff --git a/DataThread/package.json b/DataThread/package.json
@@ -1,5 +1,5 @@
 {
-    "name": "DataThread",
+    "name": "data-sharing",
     "version": "0.0.1",
     "keywords": [
         "data",
@@ -10,29 +10,26 @@
         "npm": ">=8.19.2 <9.6.5"
     },
     "bin": {
-        "graph-data": ".\\src\\graphql-server.js"
+        "graph-data": ".\\src\\main\\js\\graphql-server.js"
     },
     "scripts": {
-        "graph-data": "node ./src/graphql-server.js"
+        "graph-data": "node ./src/main/js/graphql_server.js --baseDir example/metastore"
+    },
+    "devDependencies": {
     },
     "dependencies": {
-        "@apollo/client": "^3.10.4",
-        "@typespec/compiler": "latest",
-        "@typespec/json-schema": "^0.56.0",
-        "@typespec/openapi": "^0.56.0",
+        "@apollo/client": "^3.9.11",
         "apollo-server": "^3.13.0",
         "express": "^4.19.2",
         "graphql": "^16.8.1",
         "graphql-http": "^1.22.1",
-        "graphql-request": "^7.0.1",
         "morphir-elm": "^2.89.0",
-        "ruru": "^2.0.0-beta.11",
-        "urql": "^4.1.0"
+        "ruru": "^2.0.0-beta.11"
     },
     "files": [
-        "morphir-ir.json",
-        "metastore",
-        "bin",
-        "src"
+        "morphir-ir.json"
+        ,"metastore"
+        ,"bin"
+        ,"src"
     ]
 }
diff --git a/DataThread/src/main/js/graphql_resolver.js b/DataThread/src/main/js/graphql_resolver.js
@@ -6,80 +6,84 @@ const { log } = require("console");
 
 
 class ElementResolver {
-    constructor(storage) {
-        this.storage = storage;
-    }
+  constructor(storage, elementInfoResolver) {
+      this.storage = storage;
+      this.elementInfoResolver = elementInfoResolver;
+  }
 
-    element(id) {
-      var element = this.storage.resolveAndRead(id, "element");
+  get(id) {
+    var element = this.storage.resolveAndRead(id, "element");
 
-      if(!(element === undefined) && element !== null) {
-        element = inflateElement(this.storage, element);
-      }
-
-      return element;
+    if(!(element === undefined) && element !== null) {
+      element = this.inflateElement(element);
     }
 
-    elements() {
-      const elements =
-        this.storage.findAllAndRead('element.json')
-        .map(element => inflateElement(this.storage, element));
-
-      return elements;
-    }
-}
- function inflateElement(storage, element) {
-  const elementType = element.element_type;
-    if (elementType && typeof elementType === 'object') {
-        const typeNames = {
-            'Number': 'NumberType',
-            'Reference': 'ReferenceType',
-            'Text': 'TextType',
-            'Date': 'DateType',
-            'Time': 'TimeType',
-            'DateTime': 'DateTimeType',
-            'Boolean': 'BooleanType',
-            'Enum': 'EnumType'
-        };
-
-        for (const key in typeNames) {
-            if (elementType.hasOwnProperty(key)) {
-                elementType.__typename = typeNames[key];
-
-                if (key === 'Reference') {
-                    const refId = elementType.Reference.ref;
-                    const referencedElement = storage.resolveAndRead(refId, "element");
-                    elementType.Reference.ref = inflateElement(storage, referencedElement);
-                }
-                if (key === 'Boolean') {
-                    elementType.Bool = {};
-                }
-                break;
+    return element;
+  }
+
+  getAll() {
+    const elements =
+      this.storage.findAllAndRead('element.json')
+      .map(element => this.inflateElement(element));
+
+    return elements;
+  }
+
+  inflateElement(element) {
+    const elementType = element.element_type;
+      if (elementType && typeof elementType === 'object') {
+          const typeNames = {
+              'Number': 'NumberType',
+              'Reference': 'ReferenceType',
+              'Text': 'TextType',
+              'Date': 'DateType',
+              'Time': 'TimeType',
+              'DateTime': 'DateTimeType',
+              'Boolean': 'BooleanType',
+              'Enum': 'EnumType'
+          };
+
+          for (const key in typeNames) {
+              if (elementType.hasOwnProperty(key)) {
+                  elementType.__typename = typeNames[key];
+
+                  if (key === 'Reference') {
+                      const refId = elementType.Reference.ref;
+                      const referencedElement = this.storage.resolveAndRead(refId, "element");
+                      elementType.Reference.ref = this.inflateElement(referencedElement);
+                  }
+                  if (key === 'Boolean') {
+                      elementType.Bool = {};
+                  }
+                  break;
+              }
+          }
+
+          if (!element.info) {
+            const infoOverride = this.elementInfoResolver.get(element.id);
+
+            if(infoOverride != undefined) {
+              element.info = infoOverride;
             }
-        }
-
-        if (!element.info) {
-          const infoOverride = storage.resolveAndRead(element.id, "element_info");
-          element.info = infoOverride;
-        }
-    }
+          }
+      }
 
-    return element;
+      return element;
+  }
 }
 
-
 class DatasetResolver {
     constructor(storage, elementResolver) {
         this.storage = storage;
         this.elementResolver = elementResolver;
     }
 
-     dataset(id) {
+     get(id) {
       const dataset = this.storage.resolveAndRead(id, "dataset");
       return inflateDataset(dataset);
     }
 
-    datasets() {
+    getAll() {
       const datasets = this.storage
         .findAllAndRead('dataset.json')
         .map(dataset => this.inflateDataset(dataset));
@@ -105,13 +109,13 @@ class DatasetResolver {
         if(field.element === undefined)
         {
           const elementUrn = "element" + fieldUrn;
-          const elmt = this.elementResolver.element(elementUrn);
+          const elmt = this.elementResolver.get(elementUrn);
           field.element = elmt;
         }
         else if (typeof field.element === 'string')
         {
           const elementUrn = field.element;
-          const elmt = this.elementResolver.element(elementUrn);
+          const elmt = this.elementResolver.get(elementUrn);
           field.element = elmt;
         }
         else
@@ -123,7 +127,7 @@ class DatasetResolver {
         if(field.element === undefined || field.element == null) {
           log("Setting " + field.name + " to nil in " + dataset.id);
           const elementUrn = "element:core:nil";
-          const elmt = this.elementResolver.element(elementUrn);
+          const elmt = this.elementResolver.get(elementUrn);
           field.element = elmt;
         }
 
@@ -136,8 +140,20 @@ class DatasetResolver {
     }
 }
 
+class ElementInfoResolver {
+  constructor(storage) {
+    this.storage = storage;
+  }
+
+  get(id) {
+    var item = this.storage.resolveAndRead(id, "element_info");
+    return item;
+  }
+}
+
 //// Module ////
 module.exports = {
-    DatasetResolver,
-    ElementResolver
+  ElementResolver,
+  DatasetResolver,
+  ElementInfoResolver
 };
diff --git a/DataThread/src/main/js/graphql_server.js b/DataThread/src/main/js/graphql_server.js
@@ -34,9 +34,12 @@ function runGraphQL() {
   const baseDir = path.resolve(process.cwd(), baseDirArg);
     log("Using base folder: " + baseDir);
 
-  var storage = new file_store.Storage(baseDir);
-  var  elementResolver = new gql_resolvers.ElementResolver(storage);
-  var datasetResolver = new gql_resolvers.DatasetResolver(storage, elementResolver);
+  const automatedStorage = new file_store.Storage(path.resolve(baseDir, 'automated'));
+  const editedStorage = new file_store.Storage(path.resolve(baseDir, 'edited'));
+
+  const elementInfoResolver = new gql_resolvers.ElementInfoResolver(editedStorage);
+  const elementResolver = new gql_resolvers.ElementResolver(automatedStorage, elementInfoResolver);
+  const datasetResolver = new gql_resolvers.DatasetResolver(automatedStorage, elementResolver);
 
   // Define the GraphQL schema
     const pathToGrammar = path.join(__dirname, '..', 'resources');
@@ -46,10 +49,10 @@ function runGraphQL() {
 
     const resolvers = {
       Query: {
-        dataset: (_, { id }) => datasetResolver.dataset(id),
-        datasets: () => datasetResolver.datasets(),
-        element: (_, { id }) => elementResolver.element(id),
-        elements: () => elementResolver.elements(),
+        dataset: (_, { id }) => datasetResolver.get(id),
+        datasets: () => datasetResolver.getAll(),
+        element: (_, { id }) => elementResolver.get(id),
+        elements: () => elementResolver.getAll(),
       }
     };