392 lines
15 KiB
JavaScript
392 lines
15 KiB
JavaScript
// Clears the user's data first, so it is safe to re-run.
|
|
//
|
|
// docker exec -ti \
|
|
// -e SEED_USER_ID=2 -e SEED_API_KEY=<core> -e SEED_ARCHIVE_KEY=<archive> \
|
|
// -e SEED_COUNT=1000 \
|
|
// jlinc-server node /app/scripts/seed-data.js
|
|
|
|
import axios from 'axios';
|
|
import sodium from "sodium-native";
|
|
import { createHash } from "crypto";
|
|
import stringify from 'safe-stable-stringify';
|
|
import { loadConfig } from "../common/config.js";
|
|
import { getPool, init, close } from "../db/index.js";
|
|
|
|
const config = {
|
|
userId: process.env.SEED_USER_ID,
|
|
apiUrl: 'http://localhost:9090',
|
|
apiKey: process.env.SEED_API_KEY,
|
|
archiveUrl: 'http://localhost:9090',
|
|
archiveKey: process.env.SEED_ARCHIVE_KEY,
|
|
fedidUrl: 'https://fedid-test.jlinc.io',
|
|
count: parseInt(process.env.SEED_COUNT, 10) || 1000,
|
|
}
|
|
|
|
function generateRandomString(length) {
|
|
const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
|
let result = "";
|
|
const charactersLength = characters.length;
|
|
for (let i = 0; i < length; i++) {
|
|
result += characters.charAt(Math.floor(Math.random() * charactersLength));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function generateRandomJSON() {
|
|
const firstNames = ['James', 'Mary', 'Robert', 'Patricia', 'John', 'Jennifer', 'Michael', 'Linda', 'David', 'Elizabeth', 'William', 'Barbara', 'Richard', 'Susan', 'Joseph', 'Jessica', 'Thomas', 'Sarah', 'Christopher', 'Karen'];
|
|
const lastNames = ['Smith', 'Johnson', 'Williams', 'Brown', 'Jones', 'Garcia', 'Miller', 'Davis', 'Rodriguez', 'Martinez', 'Hernandez', 'Lopez', 'Gonzalez', 'Wilson', 'Anderson', 'Thomas', 'Taylor', 'Moore', 'Jackson', 'Martin'];
|
|
const streets = ['Maple', 'Oak', 'Cedar', 'Pine', 'Elm', 'Washington', 'Main', 'Lake', 'Park', 'Church', 'Oak', 'Washington', 'Union', 'Walnut', 'Washington'];
|
|
const cities = ['Springfield', 'Riverside', 'Greenville', 'Madison', 'Lakeside', 'Franklin', 'Clinton', 'Burlington', 'Georgetown', 'Arlington', 'Salem', 'Cambridge', 'Milton', 'Medford', 'Newport'];
|
|
const states = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY'];
|
|
|
|
const pick = arr => arr[Math.floor(Math.random() * arr.length)];
|
|
const randomNum = (min, max) => Math.floor(Math.random() * (max - min + 1)) + min;
|
|
const randomDecimal = (min, max) => (Math.random() * (max - min) + min).toFixed(2);
|
|
|
|
const firstName = pick(firstNames);
|
|
const lastName = pick(lastNames);
|
|
const streetNum = randomNum(100, 9999);
|
|
const street = pick(streets);
|
|
const streetType = pick(['Street', 'Avenue', 'Boulevard', 'Drive', 'Lane', 'Court', 'Place', 'Way']);
|
|
const zip = `${randomNum(10000, 99999)}-${randomNum(1000, 9999)}`;
|
|
|
|
return {
|
|
name: `${firstName} ${lastName}`,
|
|
email: `${firstName.toLowerCase()}.${lastName.toLowerCase()}@${pick(['gmail', 'yahoo', 'outlook', 'hotmail', 'protonmail'])}.com`,
|
|
phone: `(${randomNum(200, 999)}) ${randomNum(200, 999)}-${randomNum(1000, 9999)}`,
|
|
address: {
|
|
street: `${streetNum} ${street} ${streetType}`,
|
|
city: pick(cities),
|
|
state: pick(states),
|
|
zip,
|
|
},
|
|
account: {
|
|
balance: parseFloat(randomDecimal(0, 10000)),
|
|
currency: 'USD',
|
|
status: pick(['active', 'active', 'active', 'pending', 'review']),
|
|
tier: pick(['basic', 'standard', 'premium']),
|
|
},
|
|
metadata: {
|
|
createdAt: new Date(Date.now() - randomNum(1, 365) * 86400000).toISOString().split('T')[0],
|
|
lastLogin: new Date(Date.now() - randomNum(0, 30) * 86400000).toISOString().split('T')[0],
|
|
tags: Array.from({ length: randomNum(1, 3) }, () => pick(['new', 'returning', 'premium', 'trial', 'verified', 'flagged'])),
|
|
},
|
|
};
|
|
}
|
|
|
|
const sleep = (s) => new Promise((resolve) => setTimeout(resolve, (s * 1000)));
|
|
|
|
async function api() {
|
|
const token = config.apiKey;
|
|
const archiveToken = config.archiveKey;
|
|
|
|
|
|
// Get a FedID domain
|
|
// ==================
|
|
const domains = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/entity/domains/get`,
|
|
{
|
|
fedidUrl: config.fedidUrl, // optional
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
},
|
|
)).data;
|
|
|
|
// Make the provider DID
|
|
// =====================
|
|
let provider;
|
|
provider = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/entity/create`,
|
|
{
|
|
fedidUrl: config.fedidUrl, // optional
|
|
shortName: `provider-${generateRandomString(8)}@${domains[0]}`,
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
|
|
// Create and saves an agreement that is valid once signed by the provider (create)
|
|
// Sign it, make an audit record, sign that, and deliver it (process)
|
|
const agreement = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/agreement/produce`,
|
|
{
|
|
data: {
|
|
references: [`https://sisa.jlinc.org/v1/34020c5fb59ebc6507ebca4eb38090aec1097c6aec8d2ae2250ddfed4b4aa63c`],
|
|
permitted: ['data-sharing'],
|
|
prohibited: ['non-delegable'],
|
|
shortNames: [provider.didDoc.shortName], // Anyone required to sign before the agreement is valid
|
|
validRoles: [
|
|
'provider',
|
|
'user',
|
|
'third-party',
|
|
],
|
|
// public: true,
|
|
},
|
|
shortName: provider.didDoc.shortName, // Sign this agreement with this user
|
|
role: 'provider', // Sign as this role
|
|
archive: {
|
|
url: config.archiveUrl,
|
|
key: archiveToken,
|
|
},
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
|
|
// Make the user DID
|
|
// =================
|
|
let user;
|
|
user = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/entity/create`,
|
|
{
|
|
fedidUrl: config.fedidUrl, // optional
|
|
shortName: `user-${generateRandomString(8)}@${domains[0]}`,
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
|
|
// User cross-signs the provider's signed agreement
|
|
// ================================================
|
|
// The actual signature
|
|
const processedUserAgreement = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/agreement/process`,
|
|
{
|
|
agreementId: agreement.created.agreementId,
|
|
shortName: user.didDoc.shortName,
|
|
role: 'user',
|
|
archive: {
|
|
url: config.archiveUrl,
|
|
key: archiveToken,
|
|
},
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
|
|
for (let x = 0; x < config.count; x++) {
|
|
// >>> At this point, the user is in the system, and ready to ask the chatbot questions.
|
|
|
|
// User types information into a chatbot
|
|
// =====================================
|
|
// Create and saves an event with the user's query (create)
|
|
// Sign it, make an audit record, sign that, and deliver (process)
|
|
const userEvent = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/event/produce`,
|
|
{
|
|
type: 'data',
|
|
senderShortName: user.didDoc.shortName,
|
|
recipientShortName: provider.didDoc.shortName,
|
|
agreementId: agreement.created.agreementId,
|
|
meta: {
|
|
myCustomId: 'my_custom_identifier'
|
|
},
|
|
data: generateRandomJSON(),
|
|
archive: {
|
|
url: config.archiveUrl,
|
|
key: archiveToken,
|
|
},
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
|
|
// Provider indicates they have received and will use the data
|
|
// and cross-signs the event once receiving the data
|
|
const processedProviderEvent = (await axios.post(
|
|
`${config.apiUrl}/api/v1/data/event/process`,
|
|
{
|
|
eventId: userEvent.created.eventId,
|
|
shortName: provider.didDoc.shortName,
|
|
archive: {
|
|
url: config.archiveUrl,
|
|
key: archiveToken,
|
|
},
|
|
},
|
|
{
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
}
|
|
}
|
|
)).data;
|
|
}
|
|
}
|
|
|
|
// The async archive flow lags the api() calls; poll until both counts stop
|
|
// growing, or late rows keep "now" timestamps and spike the final day.
|
|
async function waitForSettle(client) {
|
|
// Consecutive reads: one can be fooled by a pause in the archive writes.
|
|
const STABLE_NEEDED = 4;
|
|
let prevAudits = -1, prevUsage = -1, stable = 0;
|
|
for (let i = 0; i < 180; i++) {
|
|
const audits = (await client.query(
|
|
`SELECT COUNT(*)::int AS n FROM audit a JOIN event e ON e.event_id_uuid = a.event_id WHERE e.user_id = $1`,
|
|
[config.userId])).rows[0].n;
|
|
const usage = (await client.query(
|
|
`SELECT COUNT(*)::int AS n FROM usage WHERE user_id = $1`, [config.userId])).rows[0].n;
|
|
if (audits > 0 && audits === prevAudits && usage === prevUsage) {
|
|
if (++stable >= STABLE_NEEDED) return { audits, usage };
|
|
} else {
|
|
stable = 0;
|
|
}
|
|
prevAudits = audits; prevUsage = usage;
|
|
await sleep(1);
|
|
}
|
|
return { audits: prevAudits, usage: prevUsage };
|
|
}
|
|
|
|
async function dbUpdate(client) {
|
|
const settled = await waitForSettle(client);
|
|
console.log(`Settled: ${settled.audits} audit, ${settled.usage} usage record(s)`);
|
|
const userId = client.escapeLiteral(config.userId);
|
|
let sql = `
|
|
DO $$
|
|
DECLARE
|
|
r record;
|
|
BEGIN
|
|
FOR r IN SELECT ctid FROM event WHERE user_id = ${userId} LOOP
|
|
UPDATE event
|
|
SET created_as_ts = NOW() - (random() * INTERVAL '31 days')
|
|
WHERE ctid = r.ctid;
|
|
END LOOP;
|
|
END $$;
|
|
`;
|
|
await client.query(sql);
|
|
console.log(`Updated event(s)`);
|
|
sql = `
|
|
UPDATE event SET
|
|
created_ts = created_as_ts,
|
|
updated_ts = created_as_ts
|
|
WHERE user_id = $1
|
|
`;
|
|
const eventRes = await client.query(sql, [config.userId]);
|
|
console.log(`Updated ${eventRes.rowCount} event record(s)`);
|
|
sql = `
|
|
UPDATE audit
|
|
SET created_ts = e.created_ts,
|
|
updated_ts = e.created_ts
|
|
FROM event e
|
|
WHERE audit.event_id = e.event_id_uuid
|
|
AND e.user_id = $1
|
|
`;
|
|
const auditRes = await client.query(sql, [config.userId]);
|
|
console.log(`Updated ${auditRes.rowCount} audit record(s)`);
|
|
sql = `
|
|
WITH all_records AS (
|
|
-- SELECT created_ts
|
|
-- FROM audit
|
|
-- WHERE event_id IN (
|
|
-- SELECT event_id_uuid
|
|
-- FROM event
|
|
-- WHERE user_id = $1
|
|
-- )
|
|
-- UNION ALL
|
|
SELECT created_ts
|
|
FROM event
|
|
WHERE user_id = $1
|
|
),
|
|
record_counts AS (
|
|
SELECT COUNT(*) as total FROM all_records
|
|
),
|
|
numbered_all AS (
|
|
SELECT created_ts, ROW_NUMBER() OVER (ORDER BY created_ts) as rn
|
|
FROM all_records
|
|
),
|
|
usage_numbered AS (
|
|
SELECT id, ROW_NUMBER() OVER (ORDER BY id) as rn
|
|
FROM usage
|
|
WHERE user_id = $1
|
|
)
|
|
UPDATE usage
|
|
SET created_ts = (SELECT na.created_ts FROM numbered_all na WHERE na.rn = ((un.rn - 1) % (SELECT total FROM record_counts)) + 1),
|
|
updated_ts = (SELECT na.created_ts FROM numbered_all na WHERE na.rn = ((un.rn - 1) % (SELECT total FROM record_counts)) + 1)
|
|
FROM usage_numbered un
|
|
WHERE usage.id = un.id
|
|
AND usage.user_id = $1
|
|
`;
|
|
const usageRes = await client.query(sql, [config.userId]);
|
|
console.log(`Updated ${usageRes.rowCount} usage record(s)`);
|
|
|
|
}
|
|
|
|
|
|
|
|
async function dbClear(client) {
|
|
async function runClear(sql) {
|
|
await client.query(sql, [config.userId])
|
|
}
|
|
await runClear(`DELETE FROM audit_signature WHERE audit_id IN (SELECT a.audit_id FROM audit a JOIN event e ON e.event_id_uuid=a.event_id WHERE e.user_id=$1)`);
|
|
await runClear(`DELETE FROM audit_meta WHERE audit_id IN (SELECT audit_id FROM audit WHERE event_id IN (SELECT event_id_uuid FROM event WHERE user_id=$1))`);
|
|
await runClear(`DELETE FROM audit_meta WHERE audit_id IN (SELECT audit_id FROM audit WHERE agreement_id IN (SELECT agreement_id_uuid FROM agreement WHERE user_id=$1))`);
|
|
await runClear(`DELETE FROM audit WHERE event_id IN (SELECT event_id_uuid FROM event WHERE user_id=$1)`);
|
|
await runClear(`DELETE FROM signature WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM event_data WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM event_meta WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM event WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM usage WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM entity WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_purpose WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_prohibition WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_role WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_content WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_reference WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement_required_id WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM purpose WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM prohibition WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM role WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM reference WHERE user_id=$1`);
|
|
await runClear(`DELETE FROM agreement WHERE user_id=$1`);
|
|
}
|
|
|
|
async function dbInvalidate(client) {
|
|
const sql = `
|
|
WITH invalidate AS (
|
|
SELECT id
|
|
FROM event
|
|
WHERE user_id = $1
|
|
ORDER BY created_ts DESC
|
|
LIMIT 1
|
|
OFFSET 2
|
|
)
|
|
UPDATE event
|
|
SET created = EXTRACT(EPOCH FROM created_as_ts)::bigint
|
|
WHERE id = (SELECT id FROM invalidate)
|
|
|
|
`;
|
|
await client.query(sql, [config.userId]);
|
|
}
|
|
|
|
async function main() {
|
|
await loadConfig()
|
|
await init();
|
|
const client = await getPool();
|
|
|
|
await dbClear(client);
|
|
await api();
|
|
await dbUpdate(client);
|
|
await dbInvalidate(client)
|
|
|
|
await client.release();
|
|
await close();
|
|
}
|
|
|
|
main()
|