:recycle: rebuilt aspect and prescore table migrations

backend/.env.sample

@@ -14,7 +14,7 @@ DB_USER=root
 DB_NAME=test
 DB_HOST=localhost
 DB_TYPE=mysql
-DB_PORT=3306
+DB_PORT=3307
 
 PSCALE_DB_BRANCH=main
 PSCALE_DB_NAME=planet-scale-db

backend/db/data-generator/config.js

@@ -1,46 +0,0 @@
-const mockOutputPath = './db/generated'
-const magic = 1000
-
-// Insert here how many users you would like to generate:
-const total = 30
-const batchSize = 10
-
-// Amount of responses for a complete survey
-const questions = 7
-
-// Seekers per 100 profiles
-const percentageOfSeekers = 90
-
-const scoreVals = [100, 140, 180, 220, 260, 400]
-
-const header = `/**
-* GENERATED MOCK SIIMEE DATA
-* Generated at: ${Date.now()}
-*/
-`
-
-const possibleZipcodes = [
-    '90065', // Glassel
-    '90012', // Chinatown
-    '90240', // Downey
-    '91030', // South Pasadena
-    '91201', // Glendale
-    '91399', // Woodland Hills
-    '91401', // Van Nuys
-    '90840', // Long Beach
-    '91001', // Altadena
-    '91011', // La Canada Flintridge
-    '97075', // Beaverton
-]
-
-module.exports = {
-    mockOutputPath,
-    magic,
-    total,
-    batchSize,
-    questions,
-    percentageOfSeekers,
-    scoreVals,
-    header,
-    possibleZipcodes,
-}

backend/db/data-generator/config.json

@@ -0,0 +1,23 @@
+{
+    "mockOutputPath": "./db/generated",
+    "magic": 10000,
+    "total": 30,
+    "batchSize": 10,
+    "questions": 7,
+    "percentageOfSeekers": 90,
+    "scoreVals": [107, 414, 721, 1028, 1336, 1648, 2056],
+    "header": "/**\n* GENERATED MOCK SIIMEE DATA\n*/",
+    "possibleZipcodes": [
+        "90065",
+        "90012",
+        "90240",
+        "91030",
+        "91201",
+        "91399",
+        "91401",
+        "90840",
+        "91001",
+        "91011",
+        "97075"
+    ]
+}

backend/db/data-generator/generate.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -11,22 +11,22 @@
      "text": [
       "\n",
       "Scoring: ((107, 107), (1336, 1336))\n",
-      "10000\n",
+      "10000.0\n",
       "\n",
       "Scoring: ((414, 107), (414, 414))\n",
-      "8616\n",
+      "8616.0\n",
       "\n",
       "Scoring: ((414, 2056), (1648, 721))\n",
-      "5738\n",
+      "5738.0\n",
       "\n",
       "Scoring: ((1648, 721), (107, 414))\n",
-      "6173\n",
+      "6173.0\n",
       "\n",
       "Scoring: ((1336, 1648), (1028, 1648))\n",
-      "9924\n",
+      "9924.0\n",
       "\n",
       "Scoring: ((107, 414), (721, 721))\n",
-      "8616\n"
+      "8616.0\n"
      ]
     }
    ],
@@ -36,17 +36,18 @@
     "from itertools import product\n",
     "from scipy import spatial\n",
     "\n",
-    "config = {\n",
-    "    \"all_possible_responses\": [\n",
-    "        107, 414, 721, 1028, 1336, 1648, 2056\n",
-    "    ],\n",
-    "    \"aspect_size\": 2,\n",
-    "    \"magic\": 10000,\n",
-    "    \"write\": True,\n",
-    "    \"file\": \"test.js\",\n",
-    "    \"delimiter\": \"-\",\n",
-    "    \"version\": \"0.1.0\"\n",
-    "}\n",
+    " \n",
+    "f = open('./config.json')\n",
+    "from_json_config = json.load(f)\n",
+    "config = {}\n",
+    "config[\"all_possible_responses\"] = from_json_config[\"scoreVals\"]\n",
+    "config[\"magic\"] = from_json_config[\"magic\"]\n",
+    "config[\"aspect_size\"] = 2\n",
+    "config[\"write\"]= True\n",
+    "config[\"file\"] = \"../generated/prescore_matrix.json\"\n",
+    "config[\"delimiter\"] = \"-\"\n",
+    "config[\"version\"] = \"0.1.0\"\n",
+    "f.close()\n",
     "\n",
     "def createPermutations(possibilities, size):\n",
     "    return tuple(product(possibilities, repeat=size))\n",
@@ -135,10 +136,10 @@
     "    if(config[\"write\"] == True):\n",
     "        # Serializing json\n",
     "        str_m = {}\n",
-    "        for key, value in m.items():\n",
+    "        for key, values in m.items():\n",
     "            delimiter = config[\"delimiter\"]\n",
     "            str_key = delimiter.join([str(v) for v in key])\n",
-    "            str_m[str_key] = value\n",
+    "            str_m[str_key] = [int(val) for val in values]\n",
     "            str_m[\"_config\"] = config\n",
     "        json_object = json.dumps(str_m, indent = 4)\n",
     "        with open(config[\"file\"], \"w\") as file:\n",

backend/db/migrations/20220313230903_create_scores_table.js

@@ -0,0 +1,16 @@
+const matrix = require('../generated/prescore_matrix.json')
+
+exports.up = function (knex) {
+    return knex.schema.createTable('prescored_aspects', function (table) {
+        table.integer('aspect_id').primary()
+        const firstRow = Object.keys(matrix)[0]
+        colVals = matrix[firstRow]
+        colVals.forEach((col, i) => {
+            table.smallint(`${i + 1}`).notNullable()
+        })
+    })
+}
+
+exports.down = function (knex) {
+    return knex.schema.dropTable('prescored_aspects')
+}

backend/db/migrations/20220313233557_create_aspects_table.js

@@ -0,0 +1,13 @@
+const matrix = require('../generated/prescore_matrix.json')
+
+exports.up = function (knex) {
+    return knex.schema.createTable('aspect_labels', function (table) {
+        table.integer('aspect_id').primary()
+        table.smallint('a').notNullable()
+        table.smallint('b').notNullable()
+    })
+}
+
+exports.down = function (knex) {
+    return knex.schema.dropTable('aspect_labels')
+}

backend/db/seeds/01-users.js

@@ -6,7 +6,9 @@ const generatedDataPath = './db/generated'
 let fileNames = fs.readdirSync(generatedDataPath)
 for (let name of fileNames) {
     const data = require(`../generated/${name}`)
-    users = [...users, ...data.users]
+    if(name[0] == '_') {
+        users = [...users, ...data.users]
+    }
 }
 
 exports.seed = knex => {

backend/db/seeds/02-profiles.js

@@ -6,7 +6,9 @@ const generatedDataPath = './db/generated'
 let fileNames = fs.readdirSync(generatedDataPath)
 for (let name of fileNames) {
     const data = require(`../generated/${name}`)
-    profiles = [...profiles, ...data.profiles]
+    if(name[0] == '_') {
+        profiles = [...profiles, ...data.profiles]
+    }
 }
 
 exports.seed = knex => {

backend/db/seeds/03-response_keys.js

@@ -6,7 +6,7 @@ const generatedDataPath = './db/generated'
 let fileNames = fs.readdirSync(generatedDataPath)
 for (let name of fileNames) {
     const data = require(`../generated/${name}`)
-    if(data?.response_keys) {
+    if(name[0] == '_' && data?.response_keys) {
         response_keys = [...response_keys, ...data.response_keys]
     }
 }

backend/db/seeds/04-responses.js

@@ -6,7 +6,9 @@ const generatedDataPath = './db/generated'
 let fileNames = fs.readdirSync(generatedDataPath)
 for (let name of fileNames) {
     const data = require(`../generated/${name}`)
-    responses = [...responses, ...data.responses]
+    if(name[0] == '_') {
+        responses = [...responses, ...data.responses]
+    }
 }
 
 exports.seed = knex => {

backend/db/seeds/10-scores.js

@@ -0,0 +1,29 @@
+const aspects = require('../generated/prescore_matrix.json')
+
+exports.seed = function (knex) {
+    // Deletes ALL existing entries
+    return knex('prescored_aspects')
+        .truncate()
+        .then(function () {
+            let rowCount = 1
+            const makeRow = scores => {
+                scoreMap = {}
+                scores.forEach((score, i) => {
+                    scoreMap['aspect_id'] = rowCount
+                    scoreMap[i + 1] = score
+                })
+                return scoreMap
+            }
+            
+            const rows = []
+            Object.values(aspects).forEach(rowOfScores => {
+                if(Array.isArray(rowOfScores)) {
+                    const row = makeRow(rowOfScores)
+                    rows.push(row)
+                    rowCount++
+                }
+            })
+            // Inserts seed entries
+            return knex('prescored_aspects').insert(rows)
+        })
+}

backend/db/seeds/11-aspects.js

@@ -0,0 +1,25 @@
+const aspects = require('../generated/prescore_matrix.json')
+
+exports.seed = function (knex) {
+    // Deletes ALL existing entries
+    return knex('aspect_labels')
+        .truncate()
+        .then(function () {
+            let rowCount = 1
+            const rows = []
+            Object.keys(aspects).forEach(label => {
+                if(label != '_config') {
+                    const row = {
+                        aspect_id: rowCount
+                    }
+                    const ab = label.split('-')
+                    row.a = parseInt(ab[0])
+                    row.b = parseInt(ab[1])
+                    rows.push(row)
+                    rowCount++
+                }
+            })
+            // Inserts seed entries
+            return knex('aspect_labels').insert(rows)
+        })
+}

backend/knexfile.js

@@ -15,7 +15,7 @@ const pscale = {
     port: process.env.PSCALE_DB_PORT ? process.env.PSCALE_DB_PORT : 3306
 }
 
-const connectionSettings = process.env.USE_LOCAL_DB == true ? local : pscale
+const connectionSettings = process.env.USE_LOCAL_DB == "true" ? local : pscale
 
 module.exports = {
     development: {