Skip to content

Commit ccb64a3

Browse files
coodoscoodos
authored
Fix/dreamsync recommendations (#691)
* feat: switch to summary for generating wishlists * chore: add migration script * chore: add db migration * fix: wishlist filtering * chore: error logs
1 parent 2a3661b commit ccb64a3

12 files changed

Lines changed: 453 additions & 123 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ node_modules
55
.pnp
66
.pnp.js
77

8+
.docusaurus
9+
810
.env.development
911

1012
# Local env files

‎docs/src/pages/index.tsx‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@ import useDocusaurusContext from '@docusaurus/useDocusaurusContext';
33
import Layout from '@theme/Layout';
44
import { useHistory } from "@docusaurus/router"
55

6-
7-
86
export default function Home(): ReactNode {
97
const { siteConfig } = useDocusaurusContext();
108
const history = useHistory();
@@ -14,7 +12,6 @@ export default function Home(): ReactNode {
1412
history.push("/prototype/docs/Getting Started/getting-started")
1513
}, [])
1614

17-
useHistory
1815
return (
1916
<Layout
2017
title={`Hello from ${siteConfig.title}`}

‎mappings/.gitignore‎

Lines changed: 0 additions & 2 deletions
This file was deleted.

‎platforms/dreamsync-api/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
"typeorm": "typeorm-ts-node-commonjs",
1212
"migration:generate": "typeorm-ts-node-commonjs migration:generate -d src/database/data-source.ts",
1313
"migration:run": "typeorm-ts-node-commonjs migration:run -d src/database/data-source.ts",
14-
"migration:revert": "typeorm-ts-node-commonjs migration:revert -d src/database/data-source.ts"
14+
"migration:revert": "typeorm-ts-node-commonjs migration:revert -d src/database/data-source.ts",
15+
"migrate-summaries": "ts-node --project tsconfig.json scripts/migrate-summaries.ts"
1516
},
1617
"dependencies": {
1718
"axios": "^1.6.7",

‎platforms/dreamsync-api/scripts/migrate-summaries.ts‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import "reflect-metadata";
2+
import path from "node:path";
3+
import { config } from "dotenv";
4+
import { AppDataSource } from "../src/database/data-source";
5+
import { WishlistSummaryService } from "../src/services/WishlistSummaryService";
6+
import { Wishlist } from "../src/database/entities/Wishlist";
7+
8+
// Load environment variables
9+
config({ path: path.resolve(__dirname, "../../../.env") });
10+
11+
async function migrateSummaries() {
12+
console.log("Starting migration: Converting text summaries to AI-generated arrays...\n");
13+
14+
try {
15+
// Initialize database connection
16+
await AppDataSource.initialize();
17+
console.log("Database connection established\n");
18+
19+
const wishlistRepository = AppDataSource.getRepository(Wishlist);
20+
const summaryService = WishlistSummaryService.getInstance();
21+
22+
// Get all wishlists that need migration (have text summaries or no summaries)
23+
const wishlists = await wishlistRepository.find({
24+
where: { isActive: true },
25+
relations: ["user"],
26+
order: { updatedAt: "DESC" },
27+
});
28+
29+
console.log(`Found ${wishlists.length} wishlists to process\n`);
30+
31+
let processedCount = 0;
32+
let skippedCount = 0;
33+
let errorCount = 0;
34+
35+
for (const wishlist of wishlists) {
36+
try {
37+
// Check if already has array summaries
38+
const hasArraySummaries =
39+
Array.isArray(wishlist.summaryWants) && wishlist.summaryWants.length > 0 &&
40+
Array.isArray(wishlist.summaryOffers) && wishlist.summaryOffers.length > 0;
41+
42+
if (hasArraySummaries) {
43+
skippedCount++;
44+
continue;
45+
}
46+
47+
console.log(`Processing wishlist ${wishlist.id}...`);
48+
console.log(` Raw content: ${wishlist.content.substring(0, 200)}${wishlist.content.length > 200 ? '...' : ''}`);
49+
50+
// Use the summary service to generate arrays from the raw content
51+
const summary = await summaryService.summarizeWishlistContent(wishlist.content);
52+
53+
console.log(` Generated Wants: ${JSON.stringify(summary.summaryWants)}`);
54+
console.log(` Generated Offers: ${JSON.stringify(summary.summaryOffers)}`);
55+
56+
// Save the arrays to the database
57+
wishlist.summaryWants = summary.summaryWants;
58+
wishlist.summaryOffers = summary.summaryOffers;
59+
await wishlistRepository.save(wishlist);
60+
61+
console.log(` Saved successfully\n`);
62+
processedCount++;
63+
64+
} catch (error) {
65+
console.error(` Error processing wishlist ${wishlist.id}:`, error);
66+
errorCount++;
67+
}
68+
}
69+
70+
console.log("Migration completed:");
71+
console.log(` Processed: ${processedCount}`);
72+
console.log(` Skipped (already has arrays): ${skippedCount}`);
73+
console.log(` Errors: ${errorCount}\n`);
74+
75+
} catch (error) {
76+
console.error("Migration failed:", error);
77+
process.exit(1);
78+
} finally {
79+
await AppDataSource.destroy();
80+
console.log("Database connection closed");
81+
}
82+
}
83+
84+
// Run migration
85+
migrateSummaries()
86+
.then(() => {
87+
console.log("Migration script completed");
88+
process.exit(0);
89+
})
90+
.catch((error) => {
91+
console.error("Migration script failed:", error);
92+
process.exit(1);
93+
});

‎platforms/dreamsync-api/src/database/entities/Wishlist.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@ export class Wishlist {
2222
@Column({ type: "text" })
2323
content: string; // Markdown content
2424

25-
@Column({ type: "text", nullable: true })
26-
summaryWants: string | null;
25+
@Column({ type: "jsonb", nullable: true })
26+
summaryWants: string[] | null;
2727

28-
@Column({ type: "text", nullable: true })
29-
summaryOffers: string | null;
28+
@Column({ type: "jsonb", nullable: true })
29+
summaryOffers: string[] | null;
3030

3131
@Column({ type: "boolean", default: true })
3232
isActive: boolean;

‎platforms/dreamsync-api/src/database/migrations/1768904445609-migration.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { MigrationInterface, QueryRunner } from "typeorm";
2+
3+
export class Migration1768904445609 implements MigrationInterface {
4+
name = 'Migration1768904445609'
5+
6+
public async up(queryRunner: QueryRunner): Promise<void> {
7+
await queryRunner.query(`ALTER TABLE "wishlists" DROP COLUMN "summaryWants"`);
8+
await queryRunner.query(`ALTER TABLE "wishlists" ADD "summaryWants" jsonb`);
9+
await queryRunner.query(`ALTER TABLE "wishlists" DROP COLUMN "summaryOffers"`);
10+
await queryRunner.query(`ALTER TABLE "wishlists" ADD "summaryOffers" jsonb`);
11+
}
12+
13+
public async down(queryRunner: QueryRunner): Promise<void> {
14+
await queryRunner.query(`ALTER TABLE "wishlists" DROP COLUMN "summaryOffers"`);
15+
await queryRunner.query(`ALTER TABLE "wishlists" ADD "summaryOffers" text`);
16+
await queryRunner.query(`ALTER TABLE "wishlists" DROP COLUMN "summaryWants"`);
17+
await queryRunner.query(`ALTER TABLE "wishlists" ADD "summaryWants" text`);
18+
}
19+
20+
}

‎platforms/dreamsync-api/src/index.ts‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,23 +32,24 @@ AppDataSource.initialize()
3232
const exists = await platformService.checkPlatformEVaultExists();
3333

3434
if (!exists) {
35-
console.log("🔧 Creating platform eVault for DreamSync...");
35+
console.log("Creating platform eVault for DreamSync...");
3636
const result = await platformService.createPlatformEVault();
37-
console.log(`✅ Platform eVault created successfully: ${result.w3id}`);
37+
console.log(`Platform eVault created successfully: ${result.w3id}`);
3838
} else {
39-
console.log("✅ Platform eVault already exists for DreamSync");
39+
console.log("Platform eVault already exists for DreamSync");
4040
}
4141
} catch (error) {
42-
console.error("❌ Failed to initialize platform eVault:", error);
42+
console.error("Failed to initialize platform eVault:", error);
4343
// Don't exit the process, just log the error
4444
}
4545

46-
// Backfill wishlist summaries for existing records
46+
// Summarize all wishlists on platform start
4747
try {
4848
const wishlistSummaryService = WishlistSummaryService.getInstance();
49-
await wishlistSummaryService.backfillMissingSummaries();
49+
await wishlistSummaryService.summarizeAllWishlists();
5050
} catch (error) {
51-
console.error("❌ Failed to backfill wishlist summaries:", error);
51+
console.error("Failed to summarize wishlists:", error);
52+
// Don't exit the process, just log the error
5253
}
5354

5455
// Start AI matching job (disabled automatic startup)

‎platforms/dreamsync-api/src/services/AIMatchingService.ts‎

Lines changed: 43 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,26 +38,38 @@ export class AIMatchingService {
3838

3939
return withOperationContext('AIMatchingService', operationId, async () => {
4040
const wishlists = await this.getWishlistsForMatching();
41-
console.log(`📋 Found ${wishlists.length} wishlists to analyze`);
41+
console.log(`📋 Found ${wishlists.length} wishlists to analyze (after filtering blank wishlists)`);
42+
43+
if (wishlists.length === 0) {
44+
console.log("⚠️ No valid wishlists to match, skipping matching process");
45+
return;
46+
}
47+
4248
await this.ensureWishlistSummaries(wishlists);
4349

4450
// Get existing groups for context
4551
const existingGroups = await this.getExistingGroups();
4652
console.log(`🏠 Found ${existingGroups.length} existing groups to consider`);
4753

48-
// Convert to shared service format
49-
const wishlistData: WishlistData[] = wishlists.map(wishlist => ({
50-
id: wishlist.id,
51-
content: wishlist.content,
52-
summaryWants: wishlist.summaryWants || "",
53-
summaryOffers: wishlist.summaryOffers || "",
54-
userId: wishlist.userId,
55-
user: {
56-
id: wishlist.user.id,
57-
name: wishlist.user.name || wishlist.user.ename,
58-
ename: wishlist.user.ename
59-
}
60-
}));
54+
// Convert to shared service format, filtering out wishlists without summaries
55+
const wishlistData: WishlistData[] = wishlists
56+
.filter(wishlist => {
57+
// Only include wishlists with valid summary arrays
58+
return wishlist.summaryWants && wishlist.summaryWants.length > 0 &&
59+
wishlist.summaryOffers && wishlist.summaryOffers.length > 0;
60+
})
61+
.map(wishlist => ({
62+
id: wishlist.id,
63+
content: wishlist.content,
64+
summaryWants: wishlist.summaryWants || [],
65+
summaryOffers: wishlist.summaryOffers || [],
66+
userId: wishlist.userId,
67+
user: {
68+
id: wishlist.user.id,
69+
name: wishlist.user.name || wishlist.user.ename,
70+
ename: wishlist.user.ename
71+
}
72+
}));
6173

6274
// Use matching service for parallel processing
6375
const matchResults = await this.matchingService.findMatches(wishlistData, existingGroups);
@@ -224,9 +236,19 @@ export class AIMatchingService {
224236

225237
private async ensureWishlistSummaries(wishlists: Wishlist[]): Promise<void> {
226238
for (const wishlist of wishlists) {
227-
if (!wishlist.summaryWants || !wishlist.summaryOffers) {
239+
// Ensure summaries exist and are arrays
240+
if (!wishlist.summaryWants || wishlist.summaryWants.length === 0 ||
241+
!wishlist.summaryOffers || wishlist.summaryOffers.length === 0) {
228242
try {
229243
await this.wishlistSummaryService.ensureSummaries(wishlist);
244+
// Reload wishlist to get updated summaries
245+
const updated = await this.wishlistRepository.findOne({
246+
where: { id: wishlist.id },
247+
relations: ["user"]
248+
});
249+
if (updated) {
250+
Object.assign(wishlist, updated);
251+
}
230252
} catch (error) {
231253
console.error(`Failed to ensure summary for wishlist ${wishlist.id}`, error);
232254
}
@@ -383,10 +405,12 @@ Content: ${wishlistA.content}
383405
Title: ${wishlistB.title}
384406
Content: ${wishlistB.content}
385407
386-
IMPORTANT: Only return a JSON response if there's a meaningful connection (confidence > 0.7). If there's no meaningful connection, return confidence: 0 and matchType: "private" (this will be filtered out).
408+
IMPORTANT: Only return a JSON response if there's a meaningful connection (confidence > 0.85). If there's no meaningful connection, return confidence: 0 and matchType: "private" (this will be filtered out).
409+
410+
CRITICAL: If either wishlist is blank, templated with minimal content, or contains insufficient information, return confidence: 0. A blank/templated wishlist has the template structure (## What I Want / ## What I Can Do) but with very few items (2 or fewer meaningful items) or very short/placeholder content. Do NOT generate matches based on generic or placeholder content.
387411
388412
Return JSON with:
389-
1. "confidence": number between 0-1 indicating match strength (0 if no meaningful connection)
413+
1. "confidence": number between 0-1 indicating match strength (0 if no meaningful connection or if wishlists are too sparse)
390414
2. "matchType": "private" or "group" (use "private" if no connection)
391415
3. "reason": brief explanation of why they match (or why no match)
392416
4. "matchedWants": array of what User A wants that User B can offer
@@ -399,8 +423,9 @@ Consider:
399423
- Complementary needs and offerings
400424
- Potential for meaningful collaboration
401425
- Whether this could be a private connection or group activity
426+
- Whether the wishlists contain sufficient meaningful content (not just template placeholders)
402427
403-
Only suggest matches with confidence > 0.7 for meaningful connections.
428+
Only suggest matches with confidence > 0.85 for meaningful connections based on substantial content.
404429
`.trim();
405430
}
406431

0 commit comments

Comments
 (0)