// Clears the user's data first, so it is safe to re-run. // // docker exec -ti \ // -e SEED_USER_ID=2 -e SEED_API_KEY= -e SEED_ARCHIVE_KEY= \ // -e SEED_COUNT=1000 \ // jlinc-server node /app/scripts/seed-data.js import axios from 'axios'; import sodium from "sodium-native"; import { createHash } from "crypto"; import stringify from 'safe-stable-stringify'; import { loadConfig } from "../common/config.js"; import { getPool, init, close } from "../db/index.js"; const config = { userId: process.env.SEED_USER_ID, apiUrl: 'http://localhost:9090', apiKey: process.env.SEED_API_KEY, archiveUrl: 'http://localhost:9090', archiveKey: process.env.SEED_ARCHIVE_KEY, fedidUrl: 'https://fedid-test.jlinc.io', count: parseInt(process.env.SEED_COUNT, 10) || 1000, } function generateRandomString(length) { const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; let result = ""; const charactersLength = characters.length; for (let i = 0; i < length; i++) { result += characters.charAt(Math.floor(Math.random() * charactersLength)); } return result; } function generateRandomJSON() { const firstNames = ['James', 'Mary', 'Robert', 'Patricia', 'John', 'Jennifer', 'Michael', 'Linda', 'David', 'Elizabeth', 'William', 'Barbara', 'Richard', 'Susan', 'Joseph', 'Jessica', 'Thomas', 'Sarah', 'Christopher', 'Karen']; const lastNames = ['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia', 'Miller', 'Davis', 'Rodriguez', 'Martinez', 'Hernandez', 'Lopez', 'Gonzalez', 'Wilson', 'Anderson', 'Thomas', 'Taylor', 'Moore', 'Jackson', 'Martin']; const streets = ['Maple', 'Oak', 'Cedar', 'Pine', 'Elm', 'Washington', 'Main', 'Lake', 'Park', 'Church', 'Oak', 'Washington', 'Union', 'Walnut', 'Washington']; const cities = ['Springfield', 'Riverside', 'Greenville', 'Madison', 'Lakeside', 'Franklin', 'Clinton', 'Burlington', 'Georgetown', 'Arlington', 'Salem', 'Cambridge', 'Milton', 'Medford', 'Newport']; const states = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']; const pick = arr => arr[Math.floor(Math.random() * arr.length)]; const randomNum = (min, max) => Math.floor(Math.random() * (max - min + 1)) + min; const randomDecimal = (min, max) => (Math.random() * (max - min) + min).toFixed(2); const firstName = pick(firstNames); const lastName = pick(lastNames); const streetNum = randomNum(100, 9999); const street = pick(streets); const streetType = pick(['Street', 'Avenue', 'Boulevard', 'Drive', 'Lane', 'Court', 'Place', 'Way']); const zip = `${randomNum(10000, 99999)}-${randomNum(1000, 9999)}`; return { name: `${firstName} ${lastName}`, email: `${firstName.toLowerCase()}.${lastName.toLowerCase()}@${pick(['gmail', 'yahoo', 'outlook', 'hotmail', 'protonmail'])}.com`, phone: `(${randomNum(200, 999)}) ${randomNum(200, 999)}-${randomNum(1000, 9999)}`, address: { street: `${streetNum} ${street} ${streetType}`, city: pick(cities), state: pick(states), zip, }, account: { balance: parseFloat(randomDecimal(0, 10000)), currency: 'USD', status: pick(['active', 'active', 'active', 'pending', 'review']), tier: pick(['basic', 'standard', 'premium']), }, metadata: { createdAt: new Date(Date.now() - randomNum(1, 365) * 86400000).toISOString().split('T')[0], lastLogin: new Date(Date.now() - randomNum(0, 30) * 86400000).toISOString().split('T')[0], tags: Array.from({ length: randomNum(1, 3) }, () => pick(['new', 'returning', 'premium', 'trial', 'verified', 'flagged'])), }, }; } const sleep = (s) => new Promise((resolve) => setTimeout(resolve, (s * 1000))); async function api() { const token = config.apiKey; const archiveToken = config.archiveKey; // Get a FedID domain // ================== const domains = (await axios.post( `${config.apiUrl}/api/v1/data/entity/domains/get`, { fedidUrl: config.fedidUrl, // optional auth: { subject: { type: "user", id: "tester", }, action: { name: "read", }, resource: { type: "data", id: "1234", properties: { ownerID: "tester@test.com", } } } }, { headers: { 'Authorization': `Bearer ${token}`, } }, )).data; // Make the provider DID // ===================== let provider; provider = (await axios.post( `${config.apiUrl}/api/v1/data/entity/create`, { fedidUrl: config.fedidUrl, // optional shortName: `provider-${generateRandomString(8)}@${domains[0]}`, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; // Create and saves an agreement that is valid once signed by the provider (create) // Sign it, make an audit record, sign that, and deliver it (process) const agreement = (await axios.post( `${config.apiUrl}/api/v1/data/agreement/produce`, { data: { references: [`https://sisa.jlinc.org/v1/34020c5fb59ebc6507ebca4eb38090aec1097c6aec8d2ae2250ddfed4b4aa63c`], permitted: ['data-sharing'], prohibited: ['non-delegable'], shortNames: [provider.didDoc.shortName], // Anyone required to sign before the agreement is valid validRoles: [ 'provider', 'user', 'third-party', ], // public: true, }, shortName: provider.didDoc.shortName, // Sign this agreement with this user role: 'provider', // Sign as this role archive: { url: config.archiveUrl, key: archiveToken, }, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; // Make the user DID // ================= let user; user = (await axios.post( `${config.apiUrl}/api/v1/data/entity/create`, { fedidUrl: config.fedidUrl, // optional shortName: `user-${generateRandomString(8)}@${domains[0]}`, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; // User cross-signs the provider's signed agreement // ================================================ // The actual signature const processedUserAgreement = (await axios.post( `${config.apiUrl}/api/v1/data/agreement/process`, { agreementId: agreement.created.agreementId, shortName: user.didDoc.shortName, role: 'user', archive: { url: config.archiveUrl, key: archiveToken, }, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; for (let x = 0; x < config.count; x++) { // >>> At this point, the user is in the system, and ready to ask the chatbot questions. // User types information into a chatbot // ===================================== // Create and saves an event with the user's query (create) // Sign it, make an audit record, sign that, and deliver (process) const userEvent = (await axios.post( `${config.apiUrl}/api/v1/data/event/produce`, { type: 'data', senderShortName: user.didDoc.shortName, recipientShortName: provider.didDoc.shortName, agreementId: agreement.created.agreementId, meta: { myCustomId: 'my_custom_identifier' }, data: generateRandomJSON(), archive: { url: config.archiveUrl, key: archiveToken, }, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; // Provider indicates they have received and will use the data // and cross-signs the event once receiving the data const processedProviderEvent = (await axios.post( `${config.apiUrl}/api/v1/data/event/process`, { eventId: userEvent.created.eventId, shortName: provider.didDoc.shortName, archive: { url: config.archiveUrl, key: archiveToken, }, }, { headers: { 'Authorization': `Bearer ${token}`, } } )).data; } } // The async archive flow lags the api() calls; poll until both counts stop // growing, or late rows keep "now" timestamps and spike the final day. async function waitForSettle(client) { // Consecutive reads: one can be fooled by a pause in the archive writes. const STABLE_NEEDED = 4; let prevAudits = -1, prevUsage = -1, stable = 0; for (let i = 0; i < 180; i++) { const audits = (await client.query( `SELECT COUNT(*)::int AS n FROM audit a JOIN event e ON e.event_id_uuid = a.event_id WHERE e.user_id = $1`, [config.userId])).rows[0].n; const usage = (await client.query( `SELECT COUNT(*)::int AS n FROM usage WHERE user_id = $1`, [config.userId])).rows[0].n; if (audits > 0 && audits === prevAudits && usage === prevUsage) { if (++stable >= STABLE_NEEDED) return { audits, usage }; } else { stable = 0; } prevAudits = audits; prevUsage = usage; await sleep(1); } return { audits: prevAudits, usage: prevUsage }; } async function dbUpdate(client) { const settled = await waitForSettle(client); console.log(`Settled: ${settled.audits} audit, ${settled.usage} usage record(s)`); const userId = client.escapeLiteral(config.userId); let sql = ` DO $$ DECLARE r record; BEGIN FOR r IN SELECT ctid FROM event WHERE user_id = ${userId} LOOP UPDATE event SET created_as_ts = NOW() - (random() * INTERVAL '31 days') WHERE ctid = r.ctid; END LOOP; END $$; `; await client.query(sql); console.log(`Updated event(s)`); sql = ` UPDATE event SET created_ts = created_as_ts, updated_ts = created_as_ts WHERE user_id = $1 `; const eventRes = await client.query(sql, [config.userId]); console.log(`Updated ${eventRes.rowCount} event record(s)`); sql = ` UPDATE audit SET created_ts = e.created_ts, updated_ts = e.created_ts FROM event e WHERE audit.event_id = e.event_id_uuid AND e.user_id = $1 `; const auditRes = await client.query(sql, [config.userId]); console.log(`Updated ${auditRes.rowCount} audit record(s)`); sql = ` WITH all_records AS ( -- SELECT created_ts -- FROM audit -- WHERE event_id IN ( -- SELECT event_id_uuid -- FROM event -- WHERE user_id = $1 -- ) -- UNION ALL SELECT created_ts FROM event WHERE user_id = $1 ), record_counts AS ( SELECT COUNT(*) as total FROM all_records ), numbered_all AS ( SELECT created_ts, ROW_NUMBER() OVER (ORDER BY created_ts) as rn FROM all_records ), usage_numbered AS ( SELECT id, ROW_NUMBER() OVER (ORDER BY id) as rn FROM usage WHERE user_id = $1 ) UPDATE usage SET created_ts = (SELECT na.created_ts FROM numbered_all na WHERE na.rn = ((un.rn - 1) % (SELECT total FROM record_counts)) + 1), updated_ts = (SELECT na.created_ts FROM numbered_all na WHERE na.rn = ((un.rn - 1) % (SELECT total FROM record_counts)) + 1) FROM usage_numbered un WHERE usage.id = un.id AND usage.user_id = $1 `; const usageRes = await client.query(sql, [config.userId]); console.log(`Updated ${usageRes.rowCount} usage record(s)`); } async function dbClear(client) { async function runClear(sql) { await client.query(sql, [config.userId]) } await runClear(`DELETE FROM audit_signature WHERE audit_id IN (SELECT a.audit_id FROM audit a JOIN event e ON e.event_id_uuid=a.event_id WHERE e.user_id=$1)`); await runClear(`DELETE FROM audit_meta WHERE audit_id IN (SELECT audit_id FROM audit WHERE event_id IN (SELECT event_id_uuid FROM event WHERE user_id=$1))`); await runClear(`DELETE FROM audit_meta WHERE audit_id IN (SELECT audit_id FROM audit WHERE agreement_id IN (SELECT agreement_id_uuid FROM agreement WHERE user_id=$1))`); await runClear(`DELETE FROM audit WHERE event_id IN (SELECT event_id_uuid FROM event WHERE user_id=$1)`); await runClear(`DELETE FROM signature WHERE user_id=$1`); await runClear(`DELETE FROM event_data WHERE user_id=$1`); await runClear(`DELETE FROM event_meta WHERE user_id=$1`); await runClear(`DELETE FROM event WHERE user_id=$1`); await runClear(`DELETE FROM usage WHERE user_id=$1`); await runClear(`DELETE FROM entity WHERE user_id=$1`); await runClear(`DELETE FROM agreement_purpose WHERE user_id=$1`); await runClear(`DELETE FROM agreement_prohibition WHERE user_id=$1`); await runClear(`DELETE FROM agreement_role WHERE user_id=$1`); await runClear(`DELETE FROM agreement_content WHERE user_id=$1`); await runClear(`DELETE FROM agreement_reference WHERE user_id=$1`); await runClear(`DELETE FROM agreement_required_id WHERE user_id=$1`); await runClear(`DELETE FROM purpose WHERE user_id=$1`); await runClear(`DELETE FROM prohibition WHERE user_id=$1`); await runClear(`DELETE FROM role WHERE user_id=$1`); await runClear(`DELETE FROM reference WHERE user_id=$1`); await runClear(`DELETE FROM agreement WHERE user_id=$1`); } async function dbInvalidate(client) { const sql = ` WITH invalidate AS ( SELECT id FROM event WHERE user_id = $1 ORDER BY created_ts DESC LIMIT 1 OFFSET 2 ) UPDATE event SET created = EXTRACT(EPOCH FROM created_as_ts)::bigint WHERE id = (SELECT id FROM invalidate) `; await client.query(sql, [config.userId]); } async function main() { await loadConfig() await init(); const client = await getPool(); await dbClear(client); await api(); await dbUpdate(client); await dbInvalidate(client) await client.release(); await close(); } main()