From 1249a9f7966f02c6cf2d108245201fb86b2b7e5e Mon Sep 17 00:00:00 2001 From: TheUnknownHacker Date: Thu, 10 Sep 2026 18:13:28 +1000 Subject: [PATCH] remove ai fallback from journal sync when a JOURNAL.md had no # or ## headings the parser returned nothing and handed the raw file to an llm, which invented devlogs and saved them under the project owner's name with parsed hours feeding streaks and payouts. adds an audit rake task to find the projects it already hit. --- app/jobs/sync_journal_job.rb | 44 ---------- lib/tasks/audit_fabricated_devlogs.rake | 105 ++++++++++++++++++++++++ test/jobs/sync_journal_job_test.rb | 20 +++++ 3 files changed, 125 insertions(+), 44 deletions(-) create mode 100644 lib/tasks/audit_fabricated_devlogs.rake diff --git a/app/jobs/sync_journal_job.rb b/app/jobs/sync_journal_job.rb index d4c98878..d2702900 100644 --- a/app/jobs/sync_journal_job.rb +++ b/app/jobs/sync_journal_job.rb @@ -27,10 +27,6 @@ def perform(project_id, clear: false) entries = parse_journal_entries(journal_content) - if entries.empty? - entries = parse_with_ai(journal_content) - end - return if entries.empty? current_titles = entries.map { |e| e[:title] } @@ -368,44 +364,4 @@ def build_entry(title, body) time_spent: time_spent } end - - def parse_with_ai(content) - prompt = <<~PROMPT - Parse this JOURNAL.md file into individual devlog entries. Each entry should have a date/title, the content, and time spent. - - Journal content: - #{content.truncate(4000)} - - Respond in valid JSON only, no markdown fences: - [{"title": "entry title with date", "content": "the entry content", "time_spent": "time if mentioned or null"}] - PROMPT - - response = Net::HTTP.post( - URI("https://ai.hackclub.com/proxy/v1/chat/completions"), - { model: "qwen/qwen3-32b", messages: [ { role: "user", content: prompt } ] }.to_json, - "Content-Type" => "application/json", - "Authorization" => "Bearer #{ENV['HACKCLUB_AI_API_KEY']}" - ) - - if response.is_a?(Net::HTTPSuccess) - ai_content = JSON.parse(response.body).dig("choices", 0, "message", "content") || "" - json_match = ai_content.match(/\[[\s\S]*\]/) - if json_match - data = JSON.parse(json_match[0]) - return data.filter_map do |entry| - next if entry["title"].blank? - { - title: entry["title"], - content: entry["content"] || "", - time_spent: entry["time_spent"] - } - end - end - end - - [] - rescue StandardError => e - Rails.logger.error("AI journal parsing failed: #{e.message}") - [] - end end diff --git a/lib/tasks/audit_fabricated_devlogs.rake b/lib/tasks/audit_fabricated_devlogs.rake new file mode 100644 index 00000000..309e4d30 --- /dev/null +++ b/lib/tasks/audit_fabricated_devlogs.rake @@ -0,0 +1,105 @@ +def diagnose_format(content) + body = content.sub(/\A---\s*\n.*?\n---\s*\n/m, "") + levels = body.scan(/^(\#+)\s+.+$/).map { |h| h.first.length }.tally.sort.to_h + + return "empty after frontmatter" if body.strip.empty? + return "bold date lines, no headings" if levels.empty? && body.match?(/^\s*\*\*.+\*\*\s*$/) + return "plain text, no headings" if levels.empty? + return "deep headings only (#{levels.keys.join(', ')}) - parser only reads 1, 2" if (levels.keys & [ 1, 2 ]).empty? + + "headings " + levels.map { |lvl, n| "#{'#' * lvl}x#{n}" }.join(" ") +end + +namespace :devlogs do + desc "Report git-journal devlogs with no matching heading in the current JOURNAL.md" + task audit_fabricated: :environment do + delay = ENV.fetch("AUDIT_DELAY", "0.3").to_f + job = SyncJournalJob.new + + fully_fabricated = [] + partial = [] + unreachable = [] + + scope = Project.kept + .where(devlog_mode: "git") + .where.not(repo_link: [ nil, "" ]) + .where.not(status: :rejected) + .joins(:devlogs) + .distinct + + total = scope.count + puts "Auditing #{total} git-mode projects with devlogs\n\n" + + scope.find_each(batch_size: 50).with_index do |project, i| + print "\r[#{i + 1}/#{total}] project #{project.id}".ljust(60) + + parsed_repo = job.send(:parse_repo_url, project.repo_link) + unless parsed_repo + unreachable << { project: project, reason: "unparseable repo_link" } + next + end + + content = job.send(:fetch_journal, parsed_repo, project.journal_branch.presence) + if content.blank? + unreachable << { project: project, reason: "JOURNAL.md not fetchable" } + next + end + + heading_titles = job.send(:parse_journal_entries, content).map { |e| e[:title] } + devlogs = project.devlogs.to_a + orphans = devlogs.reject { |d| heading_titles.include?(d.title) } + + next if orphans.empty? + + record = { + project: project, + owner: project.user, + orphans: orphans, + heading_count: heading_titles.size, + journal_format: diagnose_format(content), + hours: orphans.sum { |d| d.time_hours.to_f } + } + + heading_titles.empty? ? fully_fabricated << record : partial << record + ensure + sleep delay if delay.positive? + end + + puts "\r".ljust(60) + + report = lambda do |label, rows| + next if rows.empty? + + puts "\n#{label} (#{rows.size} projects)" + puts "=" * 70 + rows.sort_by { |r| -r[:hours] }.each do |r| + puts "project #{r[:project].id} #{r[:project].name}" + puts " owner: #{r[:owner].display_name} <#{r[:owner].email}> tier: #{r[:project].tier} status: #{r[:project].status}" + puts " repo: #{r[:project].repo_link}" + puts " orphan devlogs: #{r[:orphans].size} headings in journal: #{r[:heading_count]} hours: #{r[:hours].round(2)}" + puts " journal format: #{r[:journal_format]}" + r[:orphans].each do |d| + flag = d.lapse_url.present? ? " [has lapse_url]" : "" + puts " ##{d.id} #{d.entry_date} #{d.time_hours.to_f.round(2)}h #{d.title.truncate(60).inspect}#{flag}" + end + puts + end + end + + report.call("FULLY FABRICATED - journal has no headings at all", fully_fabricated) + report.call("PARTIAL - some devlog titles match no heading (fabricated or renamed)", partial) + + if unreachable.any? + puts "\nCOULD NOT CHECK (#{unreachable.size} projects)" + puts "=" * 70 + unreachable.each { |u| puts " project #{u[:project].id}: #{u[:reason]}" } + end + + puts "\n" + "=" * 70 + puts "Projects audited: #{total}" + puts "Fully fabricated: #{fully_fabricated.size} projects, #{fully_fabricated.sum { |r| r[:orphans].size }} devlogs, #{fully_fabricated.sum { |r| r[:hours] }.round(2)}h" + puts "Partial orphans: #{partial.size} projects, #{partial.sum { |r| r[:orphans].size }} devlogs, #{partial.sum { |r| r[:hours] }.round(2)}h" + puts "Could not check: #{unreachable.size} projects" + puts "\nRead-only. Nothing was modified." + end +end diff --git a/test/jobs/sync_journal_job_test.rb b/test/jobs/sync_journal_job_test.rb index da1f41a5..3348a795 100644 --- a/test/jobs/sync_journal_job_test.rb +++ b/test/jobs/sync_journal_job_test.rb @@ -17,6 +17,26 @@ def run_job(journal, commit_date: nil, clear: false) job.perform(@project.id, clear: clear) end + test "a journal with no entry headings creates no devlogs" do + run_job(<<~MD, commit_date: @today) + --- + title: "Hackpad" + author: "Amy" + description: "A fairly basic macropad held togther with hopes and dreams" + created_at: "2026-09-05" + --- + MD + + assert_equal 0, @project.devlogs.count + end + + test "an empty journal creates no devlogs" do + run_job("", commit_date: @today) + run_job(" \n\n", commit_date: @today) + + assert_equal 0, @project.devlogs.count + end + test "an undated entry is credited to the day JOURNAL.md was last pushed" do run_job("# Built the enclosure\n\n**Total time spent: 2h**\n", commit_date: @today - 1)