:recycle: complete refactor of data generator | generate data in batches, and insert one file at a time

.gitignore

@@ -5,4 +5,4 @@ dist-ssr
 *.local
 **/.env
 .nyc_output
-**/_generated.js
+**/generated/*

backend/README.md

@@ -29,7 +29,8 @@ Migrating tracks changes in schema. Migration steps are contained in the `./db/m
 * Run `npm run unmigrate` to roll back one migration
 
 ### Seeding
-* Run `npm run seed` to seed the database with dummy data
+* Run `npm run generate` to generate dummy data
+* Run `npm run seed` to seed the database with generated dummy data
 
 ### Restarting
 Since we can't unseed the database, it's best to destroy the `dev` database and rebuild it.

backend/db/generator.js

@@ -1,10 +1,17 @@
 const fs = require('fs')
 const cosineSimilarity = require('compute-cosine-similarity')
-const mockOutputPath = './db/_generated.js'
+const mockOutputPath = './db/generated/_batch'
+const magic = 1000
 
 // Insert here how many users you would like to generate:
-const total = 10
+const total = 1000
+const batchSize = 100
+let batchCount = 1 // Counter to track how many things we've generated
+
 let extraProfilesToGenerate = 0
+let extraProfileCount = 0 // Counter to track how many EXTRA profiles we've generated
+
+let generatedResponseCount = 0
 
 // Amount of responses for a complete survey
 const questions = 13
@@ -12,16 +19,111 @@ const questions = 13
 // Seekers per 100 profiles
 const percentageOfSeekers = 90
 
-// Values for responses
+const scoreVals = [100, 140, 180, 220, 260, 400]
+// Values for responsess as strings
 const possibleResponses = {
-    not_important: '120',
-    some_what_important: '140',
-    important: '160',
-    very_important: '180',
-    extremely_important: '200',
-    mandatory: '400',
+    not_important: null,
+    some_what_important: null,
+    important: null,
+    very_important: null,
+    extremely_important: null,
+    mandatory: null,
+}
+for (let i = 0; i < Object.keys(possibleResponses).length; i++) {
+    const key = Object.keys(possibleResponses)[i]
+    possibleResponses[key] = scoreVals[i].toString()
+}
+
+/**
+ * Our initial file setup
+ */
+const header = `/**
+* GENERATED MOCK SIIMEE DATA
+* Generated at: ${Date.now()}
+*/
+`
+const write = async (batchNum, outputDataObject) => {
+    await fs.writeFile(`${mockOutputPath}_${batchNum}.js`, '', () => {})
+    fs.appendFile(
+        `${mockOutputPath}_${batchNum}.js`,
+        header + 'module.exports = ' + JSON.stringify(outputDataObject),
+        err => {
+            if (err) {
+                console.error(err)
+                return
+            }
+        },
+    )
 }
 
+/**
+ * [100, 140, 180, 220, 260, 400]
+ */
+const preComputedScores = {
+    100: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    140: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    180: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    220: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    260: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    400: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+}
+const score2d = (a, b) => {
+    const aScorePlusBase = [100]
+    const bScorePlusBase = [100]
+    aScorePlusBase.push(a)
+    bScorePlusBase.push(b)
+    return Math.round(
+        Math.pow(cosineSimilarity(aScorePlusBase, bScorePlusBase), 10) * magic,
+    )
+}
+scoreVals.forEach(val => {
+    scoreVals.forEach(v => {
+        preComputedScores[val][v] = score2d(val, v)
+    })
+})
+
 const possibleZipcodes = [
     '90065', // Glassel
     '90012', // Chinatown
@@ -42,7 +144,6 @@ const randomNumber = max => {
         ? 1
         : Math.floor(Math.random() * max)
 }
-
 const randomValFrom = arr => arr[randomNumber(arr.length)]
 const randomEmail = (length = 5) => {
     let chars =
@@ -74,7 +175,8 @@ const randomName = (length = 4) => {
 const generate = (classObj, amount, meta) => {
     const instances = []
     for (let i = 0; i < amount; i++) {
-        instances.push(new classObj(i + 1, meta))
+        let startFrom = meta?.starting ? meta.starting - batchSize : 0
+        instances.push(new classObj(i + startFrom + 1, meta))
     }
     return instances
 }
@@ -90,8 +192,8 @@ class User {
 }
 class Profile {
     constructor(id, override) {
-        this.user_id = override ? override.user_id : id
-        this.profile_id = override ? override.profile_id + id : id
+        this.user_id = override?.user_id ? override.user_id : id
+        this.profile_id = override?.profile_id ? override.profile_id + id : id
     }
 }
 class Response {
@@ -103,158 +205,172 @@ class Response {
     }
 }
 
-const users = generate(User, total)
-users.forEach(user => {
-    user.is_poster = randomNumber(100) > percentageOfSeekers ? 1 : 0
-    if (user.is_poster) {
-        extraProfilesToGenerate = extraProfilesToGenerate + randomNumber(2)
-    }
-    user.user_name = randomName() + ' ' + randomName()
-    user.user_email = randomEmail()
-})
-let jobPosterIds = users
-    .filter(user => user.is_poster > 0)
-    .map(user => user.user_id)
+console.log('\nStarting...\n---')
 
-// Guarentee ONE job poster
-if (!jobPosterIds.length) {
-    randomValFrom(users).is_poster = 1
-    jobPosterIds = users
+for (let batch = batchSize; batch <= total; batch += batchSize) {
+    /**
+     * Generate Users
+     */
+    let users = generate(User, batchSize, {
+        starting: batchSize * batchCount,
+    })
+    users.forEach(user => {
+        user.is_poster = randomNumber(100) > percentageOfSeekers ? 1 : 0
+        if (user.is_poster) {
+            extraProfilesToGenerate = extraProfilesToGenerate + randomNumber(2)
+        }
+        user.user_name = randomName() + ' ' + randomName()
+        user.user_email = randomEmail()
+    })
+    let jobPosterIds = users
         .filter(user => user.is_poster > 0)
         .map(user => user.user_id)
-}
+    console.log('COMPLETED: Generated Users...')
 
-const profiles = generate(Profile, total)
-
-// Generate extra job posting profiles
-// attributed to random user.is_poster === true
-// TODO: Clean this up. Hard to read...
-if (extraProfilesToGenerate > 0) {
-    let extras = []
-    for (let l = 0; l < extraProfilesToGenerate; l++) {
-        const generatedExtraProfiles = generate(Profile, 1, {
-            user_id:
-                jobPosterIds.length > 1
-                    ? randomValFrom(jobPosterIds)
-                    : jobPosterIds[0],
-            profile_id: profiles.length + l,
-        })
-        extras = [...extras, ...generatedExtraProfiles]
+    // Guarentee ONE job poster
+    if (!jobPosterIds.length) {
+        randomValFrom(users).is_poster = 1
+        jobPosterIds = users
+            .filter(user => user.is_poster > 0)
+            .map(user => user.user_id)
     }
-    extras.forEach(profile => profiles.push(profile))
-}
 
-// Generate responses, then fill in details
-const responses = generate(
-    Response,
-    (total + extraProfilesToGenerate) * questions,
-)
-profiles.forEach((profile, i) => {
-    const startingIndex = i * questions
-    for (let k = 0; k < questions; k++) {
-        const resToEdit = responses[startingIndex + k]
-        resToEdit.response_key_id = k + 1
-        resToEdit.profile_id = profile.profile_id
-        resToEdit.val =
-            k + 1 == questions
-                ? randomValFrom(possibleZipcodes)
-                : randomValFrom(Object.values(possibleResponses))
+    /**
+     * Generate Profiles
+     */
+    let profiles = generate(Profile, batchSize, {
+        starting: batchSize * batchCount,
+        profile_id: extraProfileCount,
+    })
+    // Generate extra job posting profiles
+    // attributed to random user.is_poster === true
+    // TODO: Clean this up. Hard to read...
+    if (extraProfilesToGenerate > 0) {
+        let extras = []
+        for (let l = 0; l < extraProfilesToGenerate; l++) {
+            const generatedExtraProfiles = generate(Profile, 1, {
+                user_id:
+                    jobPosterIds.length > 1
+                        ? randomValFrom(jobPosterIds)
+                        : jobPosterIds[0],
+                profile_id: batchSize * batchCount + extraProfileCount + l,
+            })
+            extras = [...extras, ...generatedExtraProfiles]
+        }
+        extras.forEach(profile => {
+            profiles.push(profile)
+            extraProfileCount++
+        })
     }
-})
-
-/**
- * Score all the profiles!
- */
-const scoreResponses = (seeker, potentialMatch) => {
-    const seekerResponses = responses
-        .filter(response => response.profile_id == seeker.profile_id)
-        .filter(response => response.val.length < 4)
-
-    const potentialMatchResponses = responses
-        .filter(response => response.profile_id == potentialMatch.profile_id)
-        .filter(response => response.val.length < 4)
+    console.log('COMPLETED: Generated Profiles...')
 
-    const checkValCb = res => {
-        const val = parseInt(res.val)
-        return isNaN(val) ? 0 : val
-    }
-    return Math.floor(
-        cosineSimilarity(
-            seekerResponses.map(checkValCb),
-            potentialMatchResponses.map(checkValCb),
-        ) * 1000,
+    /**
+     * Generate Responses
+     */
+    // Generate responses first, before filling in details
+    let responses = generate(
+        Response,
+        (batchSize + extraProfilesToGenerate) * questions,
+        { starting: generatedResponseCount + batchSize },
     )
-}
-const scoreProfile = (profile, potentialMatchList) => {
-    const scored = potentialMatchList.map(profileToCompare => {
-        return {
-            match_queue_id: null,
-            profile_id: profile.profile_id,
-            target_id: profileToCompare.profile_id,
-            is_deleted: false,
-            score: scoreResponses(profile, profileToCompare),
+    profiles.forEach((profile, i) => {
+        const startingIndex = i * questions
+        for (let k = 0; k < questions; k++) {
+            const resToEdit = responses[startingIndex + k]
+            resToEdit.response_key_id = k + 1
+            resToEdit.profile_id = profile.profile_id
+            resToEdit.val =
+                k + 1 == questions
+                    ? randomValFrom(possibleZipcodes)
+                    : randomValFrom(Object.values(possibleResponses))
         }
     })
-    return scored.sort((a, b) => a.score - b.score)
-}
-const scoreAll = () => {
-    let scores = []
-    const posterProfiles = profiles.filter(profile =>
-        jobPosterIds.includes(profile.user_id),
-    )
-    const seekerProfiles = profiles.filter(
-        profile => !jobPosterIds.includes(profile.user_id),
-    )
-    seekerProfiles.forEach(seeker => {
-        const scored = scoreProfile(seeker, posterProfiles)
-        scores = [...scored, ...scores]
-    })
-    posterProfiles.forEach(poster => {
-        const scored = scoreProfile(poster, seekerProfiles)
-        scores = [...scored, ...scores]
-    })
-    return scores.reverse()
+    generatedResponseCount = generatedResponseCount + responses.length
+    console.log('COMPLETED: Generated Responses...')
+
+    /**
+     * Our output format
+     */
+    const outputDataObject = { users, profiles, responses }
+    // const outputDataObject = { users, profiles, responses, match_queues }
+    write(batchSize * batchCount, outputDataObject)
+    batchCount++
 }
-const match_queues = scoreAll().map((score, i) => {
-    score.match_queue_id = i + 1
-    // Comment out to see the score
-    delete score.score
-    return score
-})
 
-/**
- * Our output format
- */
-const outputDataObject = { users, profiles, responses, match_queues }
+// /**
+//  * Score all the profiles!
+//  */
+// const compareProfileResponses = (seeker, potentialMatch) => {
+//     const checkValCb = res => {
+//         const val = parseInt(res.val)
+//         return isNaN(val) ? 0 : val
+//     }
+//     const filterBy = idToCheckFor => {
+//         return responses
+//             .filter(
+//                 response =>
+//                     response.profile_id == idToCheckFor &&
+//                     response.val.length < 4,
+//             )
+//             .map(checkValCb)
+//     }
+//     const seekerResponses = filterBy(seeker.profile_id)
+//     const potentialMatchResponses = filterBy(potentialMatch.profile_id)
+//     const cachedScores = []
+//     seekerResponses.forEach(seekerResponse => {
+//         potentialMatchResponses.forEach(potentialResponse => {
+//             cachedScores.push(
+//                 preComputedScores[seekerResponse][potentialResponse],
+//             )
+//         })
+//     })
+//     return Math.round(
+//         cachedScores.reduce((a, b) => a + b) / cachedScores.length,
+//     )
+// }
+// const scoreProfile = (profile, potentialMatchList) => {
+//     return potentialMatchList
+//         .map(profileToCompare => {
+//             return {
+//                 match_queue_id: null,
+//                 profile_id: profile.profile_id,
+//                 target_id: profileToCompare.profile_id,
+//                 is_deleted: false,
+//                 score: compareProfileResponses(profile, profileToCompare),
+//             }
+//         })
+//         .sort((a, b) => a.score - b.score)
+// }
 
-const jobPostings = profiles.filter(profile =>
-    jobPosterIds.includes(profile.user_id),
-).length
-const jobPosters = users.filter(user => user.is_poster > 0).length
-const header = `/**
- * GENERATED MOCK SIIMEE DATA
- * Generated at: ${Date.now()}
- * ---
- * ${jobPostings} positions listed by ${jobPosters} job posters
- * ${total + extraProfilesToGenerate - jobPostings} candidate profiles by ${
-    total + extraProfilesToGenerate - jobPostings
-} job seekers
- * ---
- * ${total + extraProfilesToGenerate} Profiles
- * ${total} Users
- */
-`
-const write = async () => {
-    await fs.writeFile(mockOutputPath, '', () => {})
-    fs.appendFile(
-        mockOutputPath,
-        header + 'module.exports = ' + JSON.stringify(outputDataObject),
-        err => {
-            if (err) {
-                console.error(err)
-                return
-            }
-        },
-    )
-}
-write()
+// const scoreAll = () => {
+//     process.stdout.write('\nScoring Profiles')
+//     let scores = []
+//     const posterProfiles = profiles.filter(profile =>
+//         jobPosterIds.includes(profile.user_id),
+//     )
+//     const seekerProfiles = profiles.filter(
+//         profile => !jobPosterIds.includes(profile.user_id),
+//     )
+//     process.stdout.write('.')
+//     for (let i = 0; i < seekerProfiles.length; i++) {
+//         const scored = scoreProfile(seekerProfiles[i], posterProfiles)
+//         scores.push(...scored)
+//     }
+//     process.stdout.write('.')
+//     for (let j = 0; j < posterProfiles.length; j++) {
+//         const scored = scoreProfile(posterProfiles[j], seekerProfiles)
+//         scores.push(...scored)
+//     }
+//     process.stdout.write('.')
+//     console.log('\n\nCOMPLETED: Scoring Profiles...')
+//     return scores.reverse()
+// }
+
+// const match_queues = scoreAll().map((score, i) => {
+//     score.match_queue_id = i + 1
+//     // Comment out  next line to see the scores
+//     delete score.score
+//     return score
+// })
+
+console.log('---\nFINISHED...\n===\n')

backend/db/seeds/01-users.js

@@ -1,14 +1,22 @@
 const mock = require('../mock')
-const generated = require('../_generated')
+const fs = require('fs')
 
-exports.seed = function (knex) {
+let users = []
+const generatedDataPath = './db/generated'
+let fileNames = fs.readdirSync(generatedDataPath)
+for (let name of fileNames) {
+    const data = require(`../generated/${name}`)
+    users = [...users, ...data.users]
+}
+
+exports.seed = knex => {
     // Deletes ALL existing entries
     return knex('users')
         .truncate()
         .then(function () {
             // Inserts seed entries
-            return generated.users
-                ? knex('users').insert(generated.users)
+            return users
+                ? knex('users').insert(users)
                 : knex('users').insert(mock.users)
         })
 }

backend/db/seeds/02-profiles.js

@@ -1,14 +1,22 @@
 const mock = require('../mock')
-const generated = require('../_generated')
+const fs = require('fs')
 
-exports.seed = function (knex) {
+let profiles = []
+const generatedDataPath = './db/generated'
+let fileNames = fs.readdirSync(generatedDataPath)
+for (let name of fileNames) {
+    const data = require(`../generated/${name}`)
+    profiles = [...profiles, ...data.profiles]
+}
+
+exports.seed = knex => {
     // Deletes ALL existing entries
     return knex('profiles')
         .truncate()
         .then(function () {
             // Inserts seed entries
-            return generated.profiles
-                ? knex('profiles').insert(generated.profiles)
+            return profiles
+                ? knex('profiles').insert(profiles)
                 : knex('profiles').insert(mock.profiles)
         })
 }

backend/db/seeds/04-responses.js

@@ -1,14 +1,22 @@
 const mock = require('../mock')
-const generated = require('../_generated')
+const fs = require('fs')
 
-exports.seed = function (knex) {
+let responses = []
+const generatedDataPath = './db/generated'
+let fileNames = fs.readdirSync(generatedDataPath)
+for (let name of fileNames) {
+    const data = require(`../generated/${name}`)
+    responses = [...responses, ...data.responses]
+}
+
+exports.seed = knex => {
     // Deletes ALL existing entries
     return knex('responses')
         .truncate()
         .then(function () {
             // Inserts seed entries
-            return generated.responses
-                ? knex('responses').insert(generated.responses)
+            return responses
+                ? knex('responses').insert(responses)
                 : knex('responses').insert(mock.responses)
         })
 }

backend/db/seeds/09-match_queues.js

@@ -1,14 +1,10 @@
 const mock = require('../mock')
-const generated = require('../_generated')
-
 exports.seed = function (knex) {
     // Deletes ALL existing entries
     return knex('match_queues')
         .truncate()
         .then(function () {
             // Inserts seed entries
-            return generated.match_queues
-                ? knex('match_queues').insert(generated.match_queues)
-                : knex('match_queues').insert(mock.match_queues)
+            return knex('match_queues').insert(mock.match_queues)
         })
 }