:recycle: breaking functions out of generator

backend/db/data-generator/classes.js

@@ -0,0 +1,29 @@
+class User {
+    constructor(id) {
+        this.user_id = id
+        this.user_name = ''
+        this.user_email = ''
+        this.is_admin = false
+        this.is_poster = false
+    }
+}
+class Profile {
+    constructor(id, override) {
+        this.user_id = override?.user_id ? override.user_id : id
+        this.profile_id = override?.profile_id ? override.profile_id + id : id
+    }
+}
+class Response {
+    constructor(id) {
+        this.response_id = id
+        this.profile_id = null
+        this.response_key_id = null
+        this.val = null
+    }
+}
+
+module.exports = {
+    User,
+    Profile,
+    Response,
+}

backend/db/data-generator/config.js

@@ -0,0 +1,46 @@
+const mockOutputPath = './db/generated/_batch'
+const magic = 1000
+
+// Insert here how many users you would like to generate:
+const total = 20
+const batchSize = 10
+
+// Amount of responses for a complete survey
+const questions = 13
+
+// Seekers per 100 profiles
+const percentageOfSeekers = 90
+
+const scoreVals = [100, 140, 180, 220, 260, 400]
+
+const header = `/**
+* GENERATED MOCK SIIMEE DATA
+* Generated at: ${Date.now()}
+*/
+`
+
+const possibleZipcodes = [
+    '90065', // Glassel
+    '90012', // Chinatown
+    '90240', // Downey
+    '91030', // South Pasadena
+    '91201', // Glendale
+    '91399', // Woodland Hills
+    '91401', // Van Nuys
+    '90840', // Long Beach
+    '91001', // Altadena
+    '91011', // La Canada Flintridge
+    '97075', // Beaverton
+]
+
+module.exports = {
+    mockOutputPath,
+    magic,
+    total,
+    batchSize,
+    questions,
+    percentageOfSeekers,
+    scoreVals,
+    header,
+    possibleZipcodes,
+}

backend/db/generator.js → backend/db/data-generator/index.js

@@ -1,25 +1,14 @@
 const fs = require('fs')
-const cosineSimilarity = require('compute-cosine-similarity')
-const mockOutputPath = './db/generated/_batch'
-const magic = 1000
+const config = require('./config')
+const random = require('./random')
+const classes = require('./classes')
+const score = require('./score')
 
-// Insert here how many users you would like to generate:
-const total = 1000
-const batchSize = 100
 let batchCount = 1 // Counter to track how many things we've generated
-
 let extraProfilesToGenerate = 0
 let extraProfileCount = 0 // Counter to track how many EXTRA profiles we've generated
+let generatedResponseCount = 0 // Counter to track every response generated
 
-let generatedResponseCount = 0
-
-// Amount of responses for a complete survey
-const questions = 13
-
-// Seekers per 100 profiles
-const percentageOfSeekers = 90
-
-const scoreVals = [100, 140, 180, 220, 260, 400]
 // Values for responsess as strings
 const possibleResponses = {
     not_important: null,
@@ -31,22 +20,17 @@ const possibleResponses = {
 }
 for (let i = 0; i < Object.keys(possibleResponses).length; i++) {
     const key = Object.keys(possibleResponses)[i]
-    possibleResponses[key] = scoreVals[i].toString()
+    possibleResponses[key] = config.scoreVals[i].toString()
 }
 
 /**
  * Our initial file setup
  */
-const header = `/**
-* GENERATED MOCK SIIMEE DATA
-* Generated at: ${Date.now()}
-*/
-`
 const write = async (batchNum, outputDataObject) => {
-    await fs.writeFile(`${mockOutputPath}_${batchNum}.js`, '', () => {})
+    await fs.writeFile(`${config.mockOutputPath}_${batchNum}.js`, '', () => {})
     fs.appendFile(
-        `${mockOutputPath}_${batchNum}.js`,
-        header + 'module.exports = ' + JSON.stringify(outputDataObject),
+        `${config.mockOutputPath}_${batchNum}.js`,
+        config.header + 'module.exports = ' + JSON.stringify(outputDataObject),
         err => {
             if (err) {
                 console.error(err)
@@ -56,190 +40,44 @@ const write = async (batchNum, outputDataObject) => {
     )
 }
 
-/**
- * [100, 140, 180, 220, 260, 400]
- */
-const preComputedScores = {
-    100: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-    140: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-    180: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-    220: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-    260: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-    400: {
-        100: 0,
-        140: 0,
-        180: 0,
-        220: 0,
-        260: 0,
-        400: 0,
-    },
-}
-const score2d = (a, b) => {
-    const aScorePlusBase = [100]
-    const bScorePlusBase = [100]
-    aScorePlusBase.push(a)
-    bScorePlusBase.push(b)
-    return Math.round(
-        Math.pow(cosineSimilarity(aScorePlusBase, bScorePlusBase), 10) * magic,
-    )
-}
-scoreVals.forEach(val => {
-    scoreVals.forEach(v => {
-        preComputedScores[val][v] = score2d(val, v)
-    })
-})
-
-const possibleZipcodes = [
-    '90065', // Glassel
-    '90012', // Chinatown
-    '90240', // Downey
-    '91030', // South Pasadena
-    '91201', // Glendale
-    '91399', // Woodland Hills
-    '91401', // Van Nuys
-    '90840', // Long Beach
-    '91001', // Altadena
-    '91011', // La Canada Flintridge
-    '97075', // Beaverton
-]
-
 // Helper functions
-const randomNumber = max => {
-    return Math.floor(Math.random() * max) < 1
-        ? 1
-        : Math.floor(Math.random() * max)
-}
-const randomValFrom = arr => arr[randomNumber(arr.length)]
-const randomEmail = (length = 5) => {
-    let chars =
-        'abcdefghijklmnopqrstuvwxyz-_abcdefghijklmnopqrstuvwxyz0123456789'
-    let str = ''
-    for (let i = 0; i < length + randomNumber(9); i++) {
-        str += chars.charAt(Math.floor(Math.random() * chars.length))
-    }
-    const suffixs = [
-        '@gmail.com',
-        '@aol.com',
-        '@yahoo.com',
-        '@apple.com',
-        '@hotmail.com',
-        '@rocket-mail.com',
-        '@mail.com',
-    ]
-    return str + randomValFrom(suffixs)
-}
-const randomName = (length = 4) => {
-    let chars = 'aeiouaeiouabcdefghijklmnoprstuvwyabcdefghijklmnopqrstuvwxyz'
-    let str = ''
-    for (let i = 0; i < length + randomNumber(9); i++) {
-        str += chars.charAt(Math.floor(Math.random() * chars.length))
-    }
-    return str
-}
-
+// const preComputedScores = score.precomputed
 const generate = (classObj, amount, meta) => {
     const instances = []
     for (let i = 0; i < amount; i++) {
-        let startFrom = meta?.starting ? meta.starting - batchSize : 0
+        let startFrom = meta?.starting ? meta.starting - config.batchSize : 0
         instances.push(new classObj(i + startFrom + 1, meta))
     }
     return instances
 }
 
-class User {
-    constructor(id) {
-        this.user_id = id
-        this.user_name = ''
-        this.user_email = ''
-        this.is_admin = false
-        this.is_poster = false
-    }
-}
-class Profile {
-    constructor(id, override) {
-        this.user_id = override?.user_id ? override.user_id : id
-        this.profile_id = override?.profile_id ? override.profile_id + id : id
-    }
-}
-class Response {
-    constructor(id) {
-        this.response_id = id
-        this.profile_id = null
-        this.response_key_id = null
-        this.val = null
-    }
-}
-
 console.log('\nStarting...\n---')
 
-for (let batch = batchSize; batch <= total; batch += batchSize) {
-    /**
-     * Generate Users
-     */
-    let users = generate(User, batchSize, {
-        starting: batchSize * batchCount,
+/**
+ * Generate Users
+ */
+const generateUsers = () => {
+    let users = generate(classes.User, config.batchSize, {
+        starting: config.batchSize * batchCount,
     })
     users.forEach(user => {
-        user.is_poster = randomNumber(100) > percentageOfSeekers ? 1 : 0
+        user.is_poster = random.number(100) > config.percentageOfSeekers ? 1 : 0
         if (user.is_poster) {
-            extraProfilesToGenerate = extraProfilesToGenerate + randomNumber(2)
+            extraProfilesToGenerate = extraProfilesToGenerate + random.number(2)
         }
-        user.user_name = randomName() + ' ' + randomName()
-        user.user_email = randomEmail()
+        user.user_name = random.name() + ' ' + random.name()
+        user.user_email = random.email()
     })
-    let jobPosterIds = users
-        .filter(user => user.is_poster > 0)
-        .map(user => user.user_id)
     console.log('COMPLETED: Generated Users...')
+    return users
+}
 
-    // Guarentee ONE job poster
-    if (!jobPosterIds.length) {
-        randomValFrom(users).is_poster = 1
-        jobPosterIds = users
-            .filter(user => user.is_poster > 0)
-            .map(user => user.user_id)
-    }
-
-    /**
-     * Generate Profiles
-     */
-    let profiles = generate(Profile, batchSize, {
-        starting: batchSize * batchCount,
+/**
+ * Generate Profiles
+ */
+const generateProfiles = jobPosterIds => {
+    let profiles = generate(classes.Profile, config.batchSize, {
+        starting: config.batchSize * batchCount,
         profile_id: extraProfileCount,
     })
     // Generate extra job posting profiles
@@ -248,12 +86,13 @@ for (let batch = batchSize; batch <= total; batch += batchSize) {
     if (extraProfilesToGenerate > 0) {
         let extras = []
         for (let l = 0; l < extraProfilesToGenerate; l++) {
-            const generatedExtraProfiles = generate(Profile, 1, {
+            const generatedExtraProfiles = generate(classes.Profile, 1, {
                 user_id:
                     jobPosterIds.length > 1
-                        ? randomValFrom(jobPosterIds)
+                        ? random.valFrom(jobPosterIds)
                         : jobPosterIds[0],
-                profile_id: batchSize * batchCount + extraProfileCount + l,
+                profile_id:
+                    config.batchSize * batchCount + extraProfileCount + l,
             })
             extras = [...extras, ...generatedExtraProfiles]
         }
@@ -263,43 +102,67 @@ for (let batch = batchSize; batch <= total; batch += batchSize) {
         })
     }
     console.log('COMPLETED: Generated Profiles...')
+    return profiles
+}
 
-    /**
-     * Generate Responses
-     */
+/**
+ * Generate Responses
+ */
+const generateResponses = profiles => {
     // Generate responses first, before filling in details
     let responses = generate(
-        Response,
-        (batchSize + extraProfilesToGenerate) * questions,
-        { starting: generatedResponseCount + batchSize },
+        classes.Response,
+        (config.batchSize + extraProfilesToGenerate) * config.questions,
+        { starting: generatedResponseCount + config.batchSize },
     )
     profiles.forEach((profile, i) => {
-        const startingIndex = i * questions
-        for (let k = 0; k < questions; k++) {
+        const startingIndex = i * config.questions
+        for (let k = 0; k < config.questions; k++) {
             const resToEdit = responses[startingIndex + k]
             resToEdit.response_key_id = k + 1
             resToEdit.profile_id = profile.profile_id
             resToEdit.val =
-                k + 1 == questions
-                    ? randomValFrom(possibleZipcodes)
-                    : randomValFrom(Object.values(possibleResponses))
+                k + 1 == config.questions
+                    ? random.valFrom(config.possibleZipcodes)
+                    : random.valFrom(Object.values(possibleResponses))
         }
     })
     generatedResponseCount = generatedResponseCount + responses.length
     console.log('COMPLETED: Generated Responses...')
+    return responses
+}
+
+/**
+ * Our main generator loop
+ */
+for (
+    let batch = config.batchSize;
+    batch <= config.total;
+    batch += config.batchSize
+) {
+    const users = generateUsers()
 
-    /**
-     * Our output format
-     */
-    const outputDataObject = { users, profiles, responses }
-    // const outputDataObject = { users, profiles, responses, match_queues }
-    write(batchSize * batchCount, outputDataObject)
+    let jobPosterIds = users
+        .filter(user => user.is_poster > 0)
+        .map(user => user.user_id)
+    // Guarentee ONE job poster
+    if (!jobPosterIds.length) {
+        random.valFrom(users).is_poster = 1
+        jobPosterIds = users
+            .filter(user => user.is_poster > 0)
+            .map(user => user.user_id)
+    }
+
+    const profiles = generateProfiles(jobPosterIds)
+    const responses = generateResponses(profiles)
+
+    write(config.batchSize * batchCount, { users, profiles, responses })
     batchCount++
 }
 
-// /**
-//  * Score all the profiles!
-//  */
+/**
+ * Score all the profiles!
+ */
 // const compareProfileResponses = (seeker, potentialMatch) => {
 //     const checkValCb = res => {
 //         const val = parseInt(res.val)

backend/db/data-generator/random.js

@@ -0,0 +1,39 @@
+const randomNumber = max => {
+    return Math.floor(Math.random() * max) < 1
+        ? 1
+        : Math.floor(Math.random() * max)
+}
+const randomValFrom = arr => arr[randomNumber(arr.length)]
+const randomEmail = (length = 5) => {
+    let chars =
+        'abcdefghijklmnopqrstuvwxyz-_abcdefghijklmnopqrstuvwxyz0123456789'
+    let str = ''
+    for (let i = 0; i < length + randomNumber(9); i++) {
+        str += chars.charAt(Math.floor(Math.random() * chars.length))
+    }
+    const suffixs = [
+        '@gmail.com',
+        '@aol.com',
+        '@yahoo.com',
+        '@apple.com',
+        '@hotmail.com',
+        '@rocket-mail.com',
+        '@mail.com',
+    ]
+    return str + randomValFrom(suffixs)
+}
+const randomName = (length = 4) => {
+    let chars = 'aeiouaeiouabcdefghijklmnoprstuvwyabcdefghijklmnopqrstuvwxyz'
+    let str = ''
+    for (let i = 0; i < length + randomNumber(9); i++) {
+        str += chars.charAt(Math.floor(Math.random() * chars.length))
+    }
+    return str
+}
+
+module.exports = {
+    number: randomNumber,
+    valFrom: randomValFrom,
+    email: randomEmail,
+    name: randomName,
+}

backend/db/data-generator/score.js

@@ -0,0 +1,73 @@
+const cosineSimilarity = require('compute-cosine-similarity')
+
+const config = require('./config')
+
+const preComputedScores = {
+    100: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    140: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    180: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    220: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    260: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+    400: {
+        100: 0,
+        140: 0,
+        180: 0,
+        220: 0,
+        260: 0,
+        400: 0,
+    },
+}
+const score2d = (a, b) => {
+    const aScorePlusBase = [100]
+    const bScorePlusBase = [100]
+    aScorePlusBase.push(a)
+    bScorePlusBase.push(b)
+    return Math.round(
+        Math.pow(cosineSimilarity(aScorePlusBase, bScorePlusBase), 10) *
+            config.magic,
+    )
+}
+config.scoreVals.forEach(val => {
+    config.scoreVals.forEach(v => {
+        preComputedScores[val][v] = score2d(val, v)
+    })
+})
+
+module.exports = {
+    precomputed: preComputedScores,
+}

backend/package.json

@@ -7,7 +7,7 @@
     "start": "nodemon server",
     "migrate": "knex migrate:latest",
     "unmigrate": "knex migrate:down",
-    "generate": "node ./db/generator.js",
+    "generate": "node ./db/data-generator/index.js",
     "seed": "knex seed:run",
     "test": "nyc ava --timeout=3000"
   },