Exporting my Pieces code snippets to Obsidian

#!/usr/bin/env ruby
# frozen_string_literal: true

require "date"
require "digest"
require "fileutils"
require "json"
require "optparse"
require "securerandom"
require "time"

EXPECTED_ASSET_COUNT = 358

REAL_LUA_ASSET_ID = "eb6ef70c-fd33-47a1-967e-7c6e246384f6"

LANGUAGE_OVERRIDES = {
  "c379d282-ca8b-493e-a4df-fa010ebc3797" => "javascript",
  "c8ecd0c3-693e-4026-8698-8185e20c4d0b" => "bash",
  "f943f133-cb73-4b47-a5c3-0078712906bf" => "json",
  "93cf8228-2aae-4691-8525-f6e37cecd189" => "yaml",
  "34d06144-5932-47b0-b00d-3e55b9ea8d19" => "markdown",
  "e46af71d-21f2-49b5-bc02-4b5021fd0681" => "zsh",
  "1f8a754f-e5ec-418b-b4b2-92bd24aa8798" => "shell",
  "0af3ccde-f9c3-409b-8171-2f01db9b869e" => "shell",
  "cddeb072-f13d-4b7b-94a9-1a1e8c480775" => "bash",
  "51c77565-8545-4ce7-86b9-476c962a7376" => "bash",
  "64cb2e46-129e-44b5-af47-deb67bc4870b" => "json",
  "12f44507-75ae-48a1-aada-77c75e9b4814" => "graphql",
  "d16236f4-5d05-4fc5-8465-4df119a7d0b3" => "graphql"
}.freeze

ANALYSIS_LANGUAGE_MAP = {
  "ASP" => "asp",
  "BatchFile" => "batch",
  "CSS" => "css",
  "CoffeeScript" => "coffeescript",
  "Elixir" => "elixir",
  "Groovy" => "groovy",
  "HTML" => "html",
  "JSON" => "json",
  "JavaScript" => "javascript",
  "Lua" => "lua",
  "Markdown" => "markdown",
  "Perl" => "perl",
  "Python" => "python",
  "R" => "r",
  "Ruby" => "ruby",
  "SQL" => "sql",
  "Shell" => "shell",
  "TEX" => "tex",
  "TOML" => "toml",
  "TypeScript" => "typescript",
  "YAML" => "yaml"
}.freeze

CLASSIFICATION_LANGUAGE_MAP = {
  "asp" => "asp",
  "bat" => "batch",
  "bash" => "bash",
  "coffee" => "coffeescript",
  "css" => "css",
  "graphql" => "graphql",
  "groovy" => "groovy",
  "html" => "html",
  "js" => "javascript",
  "json" => "json",
  "lua" => "lua",
  "md" => "markdown",
  "pl" => "perl",
  "py" => "python",
  "r" => "r",
  "rb" => "ruby",
  "sh" => "shell",
  "sql" => "sql",
  "tex" => "tex",
  "toml" => "toml",
  "ts" => "typescript",
  "yaml" => "yaml",
  "yml" => "yaml",
  "zsh" => "zsh"
}.freeze

FENCE_LANGUAGE_MAP = {
  "applescript" => "applescript",
  "asp" => "asp",
  "bash" => "bash",
  "batch" => "batch",
  "coffeescript" => "coffeescript",
  "css" => "css",
  "elixir" => "elixir",
  "graphql" => "graphql",
  "groovy" => "groovy",
  "html" => "html",
  "javascript" => "javascript",
  "json" => "json",
  "lua" => "lua",
  "markdown" => "markdown",
  "perl" => "perl",
  "python" => "python",
  "r" => "r",
  "ruby" => "ruby",
  "shell" => "bash",
  "sql" => "sql",
  "tex" => "tex",
  "toml" => "toml",
  "typescript" => "typescript",
  "yaml" => "yaml",
  "zsh" => "zsh"
}.freeze

KNOWN_CONTROL_CHARACTERS = {
  "c379d282-ca8b-493e-a4df-fa010ebc3797" => [0x07, 0x1B]
}.freeze

options = {
  input: "./tmp/pieces-assets.json",
  output: "./snippets",
  force: false
}

OptionParser.new do |parser|
  parser.banner = "Usage: ruby migrate_pieces_to_obsidian.rb [options]"

  parser.on("-i", "--input PATH", "Pieces assets.json snapshot") do |path|
    options[:input] = path
  end

  parser.on("-o", "--output PATH", "Output directory") do |path|
    options[:output] = path
  end

  parser.on("-f", "--force", "Replace an existing output directory") do
    options[:force] = true
  end
end.parse!

input_path = File.expand_path(options[:input])
output_path = File.expand_path(options[:output])

abort "Input does not exist: #{input_path}" unless File.file?(input_path)

if File.exist?(output_path) && !options[:force]
  abort <<~MESSAGE
    Output already exists: #{output_path}
    Re-run with --force to replace it.
  MESSAGE
end

def extract_body(asset)
  raw = asset.dig("original", "reference", "fragment", "string", "raw")

  if raw.is_a?(String)
    return [raw.dup.force_encoding(Encoding::UTF_8), "string"]
  end

  bytes = asset.dig("original", "reference", "file", "bytes", "raw")

  if bytes.is_a?(Array)
    unless bytes.all? { |byte| byte.is_a?(Integer) && byte.between?(0, 255) }
      raise "Invalid byte array for asset #{asset.fetch("id")}"
    end

    return [
      bytes.pack("C*").force_encoding(Encoding::UTF_8),
      "file_bytes"
    ]
  end

  raise "Unsupported body storage for asset #{asset.fetch("id")}"
end

def validate_body!(asset, body)
  id = asset.fetch("id")

  raise "Invalid UTF-8 in asset #{id}" unless body.valid_encoding?

  if body.bytes.first(3) == [0xEF, 0xBB, 0xBF]
    raise "Unexpected UTF-8 BOM in asset #{id}"
  end

  controls = body.codepoints.select do |codepoint|
    (codepoint < 0x20 && ![0x09, 0x0A, 0x0D].include?(codepoint)) ||
      codepoint == 0x7F
  end

  allowed = KNOWN_CONTROL_CHARACTERS.fetch(id, [])
  unexpected = controls.uniq - allowed

  return if unexpected.empty?

  formatted = unexpected.map { |cp| "U+%04X" % cp }.join(", ")

  raise "Unexpected control characters in asset #{id}: #{formatted}"
end

def source_timestamp(asset, key)
  value = asset.dig(key, "value")

  unless value.is_a?(String)
    raise "Missing #{key}.value for asset #{asset.fetch("id")}"
  end

  Time.iso8601(value)
  value
end

def analysis_language(asset)
  asset.dig("original", "reference", "analysis", "code", "language") ||
    asset.dig("original", "analysis", "code", "language") ||
    asset.dig("analysis", "code", "language")
end

def classification_language(asset)
  asset.dig("original", "reference", "classification", "specific") ||
    asset.dig("original", "classification", "specific") ||
    asset.dig("classification", "specific")
end

def shell_like?(body)
  patterns = [
    /\A\s*#!/,
    /\b(?:mdfind|mdls|defaults|osascript|plutil|brew|git|gh|curl|wget|grep|sed|awk|find|fd|fzf|xargs|open|launchctl|pg_ctl)\b/,
    /\b(?:if|then|elif|fi|for|while|until|do|done|case|esac|function)\b/,
    /\$\{?[A-Za-z_][A-Za-z0-9_]*\}?/,
    /(?:&&|\|\|)/,
    /\|\s*[A-Za-z_][A-Za-z0-9_-]*/
  ]

  patterns.any? { |pattern| body.match?(pattern) }
end

def normalize_language(asset, language)
  case language
  when "asp"
    "markdown"
  when "groovy"
    "graphql"
  when "tex"
    "shell"
  when "lua"
    asset.fetch("id") == REAL_LUA_ASSET_ID ? "lua" : "applescript"
  else
    language
  end
end

def resolve_language(asset, body)
  id = asset.fetch("id")

  if LANGUAGE_OVERRIDES.key?(id)
    return LANGUAGE_OVERRIDES.fetch(id)
  end

  analysis = analysis_language(asset)
  specific = classification_language(asset)

  detected =
    if analysis.is_a?(String) &&
       !analysis.empty? &&
       !%w[Unknown UNKNOWN TEXT text].include?(analysis)
      ANALYSIS_LANGUAGE_MAP[analysis]
    end

  if detected.nil? &&
     specific.is_a?(String) &&
     !specific.empty? &&
     !%w[Unknown UNKNOWN TEXT text].include?(specific)
    detected = CLASSIFICATION_LANGUAGE_MAP[specific.downcase]
  end

  # Pieces sometimes identifies Unix shell snippets as Windows BatchFile.
  if detected == "batch" && shell_like?(body)
    detected = "shell"
  end

  normalize_language(asset, detected)
end

def manual_tags(asset)
  id = asset.fetch("id")

  Array(asset.dig("tags", "iterable"))
    .select do |tag|
      text = tag["text"]

      text.is_a?(String) &&
        !text.strip.empty? &&
        tag.dig("mechanisms", id) == "MANUAL"
    end
    .map { |tag| tag.fetch("text").strip }
    .uniq
end

def manual_explanation(asset)
  annotations = Array(asset.dig("annotations", "iterable"))

  candidates = annotations.select do |annotation|
    annotation["type"] == "EXPLANATION" &&
      annotation["mechanism"] == "MANUAL" &&
      annotation["text"].is_a?(String) &&
      !annotation["text"].strip.empty?
  end

  candidates.max_by do |annotation|
    annotation.dig("updated", "value") ||
      annotation.dig("created", "value") ||
      ""
  end&.fetch("text")&.strip
end

def accepted_source_url(asset)
  id = asset.fetch("id")

  Array(asset.dig("websites", "iterable"))
    .find do |website|
      url = website["url"].to_s

      website.dig("mechanisms", id) == "MANUAL" &&
        website["name"] == "Origin Website" &&
        !url.empty? &&
        !url.include?("cache.raindrop.io") &&
        !url.include?("X-Amz-Signature=") &&
        !url.include?("X-Amz-Expires=") &&
        !url.include?("/_next/static/chunks/")
    end
    &.fetch("url")
end

def sanitize_filename(name)
  value = name.to_s.unicode_normalize(:nfc)

  value = value
    # macOS / path separators plus characters Obsidian disallows.
    .gsub(/[\/:\[\]#^|]+/, " - ")
    .gsub(/[\u0000-\u001F\u007F]/, " ")
    .gsub(/\s+-\s+-\s+/, " - ")
    .gsub(/\s+/, " ")
    .strip
    .sub(/\A[.\s]+/, "")
    .sub(/[.\s]+\z/, "")

  value = "Untitled" if value.empty?
  value
end

def collision_key(filename)
  filename
    .unicode_normalize(:nfc)
    .downcase
end

def safe_fence(body)
  max_run = body.scan(/`+/).map(&:length).max || 0
  "`" * [3, max_run + 1].max
end

def yaml_string(value)
  JSON.generate(value)
end

def frontmatter(
  language:,
  tags:,
  source_url:,
  created_at:,
  updated_at:
)
  output = +"---\n"
  output << "type: snippet\n"
  output << %(source: "[[Pieces.app]]"\n)
  output << "language: #{yaml_string(language)}\n" if language
  output << "created_on: #{Date.parse(created_at)}\n"
  output << "updated_on: #{Date.parse(updated_at)}\n"

  unless tags.empty?
    output << "tags:\n"

    tags.each do |tag|
      output << "  - #{yaml_string(tag)}\n"
    end
  end

  if source_url
    output << "source_url: #{yaml_string(source_url)}\n"
  end

  output << "---\n"
  output
end

def markdown_for(
  body:,
  language:,
  tags:,
  source_url:,
  explanation:,
  created_at:,
  updated_at:
)
  markdown = +""

  markdown << frontmatter(
    language: language,
    tags: tags,
    source_url: source_url,
    created_at: created_at,
    updated_at: updated_at
  )

  if explanation
    markdown << "\n## Explanation\n\n"
    markdown << explanation
    markdown << "\n"
  end

  markdown << "\n## Code\n\n"

  fence = safe_fence(body)
  fence_language = FENCE_LANGUAGE_MAP.fetch(language, language).to_s

  markdown << fence
  markdown << fence_language unless fence_language.empty?
  markdown << "\n"
  markdown << body

  # The closing Markdown fence must begin on its own line. The manifest
  # retains whether the original Pieces body actually had a trailing newline.
  markdown << "\n" unless body.end_with?("\n")

  markdown << fence
  markdown << "\n"

  markdown
end

raw_json = File.binread(input_path)
raw_sha256 = Digest::SHA256.hexdigest(raw_json)

data = JSON.parse(raw_json)
assets = data.fetch("iterable")

unless assets.length == EXPECTED_ASSET_COUNT
  abort(
    "Expected #{EXPECTED_ASSET_COUNT} assets from the audited snapshot, " \
    "but found #{assets.length}. Re-audit before exporting."
  )
end

asset_ids = assets.map { |asset| asset.fetch("id") }

unless asset_ids.uniq.length == asset_ids.length
  abort "Duplicate Pieces asset IDs detected."
end

original_ids = assets.map { |asset| asset.dig("original", "id") }

unless original_ids.compact.uniq.length == original_ids.compact.length
  abort "Duplicate Pieces original IDs detected."
end

prepared = assets.map do |asset|
  body, storage = extract_body(asset)
  validate_body!(asset, body)

  created_at = source_timestamp(asset, "created")
  updated_at = source_timestamp(asset, "updated")

  if Time.iso8601(updated_at) < Time.iso8601(created_at)
    raise "updated < created for asset #{asset.fetch("id")}"
  end

  {
    asset: asset,
    id: asset.fetch("id"),
    original_id: asset.dig("original", "id"),
    name: asset.fetch("name"),
    basename: sanitize_filename(asset.fetch("name")),
    body: body,
    body_storage: storage,
    body_sha256: Digest::SHA256.hexdigest(body.b),
    body_bytes: body.bytesize,
    trailing_newline: body.end_with?("\n"),
    language: resolve_language(asset, body),
    raw_analysis_language: analysis_language(asset),
    raw_classification_language: classification_language(asset),
    created_at: created_at,
    updated_at: updated_at,
    tags: manual_tags(asset),
    explanation: manual_explanation(asset),
    source_url: accepted_source_url(asset)
  }
end

groups = prepared.group_by do |record|
  collision_key(record.fetch(:basename))
end

prepared.each do |record|
  colliding =
    groups.fetch(
      collision_key(record.fetch(:basename))
    ).length > 1

  record[:filename] =
    if colliding
      "#{record.fetch(:basename)}--#{record.fetch(:id)[0, 8]}.md"
    else
      "#{record.fetch(:basename)}.md"
    end
end

filenames = prepared.map do |record|
  collision_key(record.fetch(:filename))
end

unless filenames.uniq.length == filenames.length
  abort "Filename collision remains after deterministic ID suffixing."
end

temporary_path =
  "#{output_path}.tmp-#{Process.pid}-#{SecureRandom.hex(4)}"

begin
  FileUtils.rm_rf(temporary_path)

  raw_dir = File.join(temporary_path, "raw")
  normalized_dir = File.join(temporary_path, "normalized")
  markdown_dir = File.join(temporary_path, "markdown")

  FileUtils.mkdir_p(raw_dir)
  FileUtils.mkdir_p(normalized_dir)
  FileUtils.mkdir_p(markdown_dir)

  raw_output = File.join(raw_dir, "assets.json")

  File.binwrite(raw_output, raw_json)
  File.chmod(0o600, raw_output)

  normalized_path =
    File.join(normalized_dir, "assets.jsonl")

  File.open(normalized_path, "wb") do |file|
    prepared.sort_by { |record| record.fetch(:id) }.each do |record|
      asset = record.fetch(:asset)

      normalized = {
        id: record.fetch(:id),
        original_id: record[:original_id],
        name: record.fetch(:name),
        filename: record.fetch(:filename),

        body: record.fetch(:body),
        body_storage: record.fetch(:body_storage),
        body_sha256: record.fetch(:body_sha256),
        body_bytes: record.fetch(:body_bytes),
        trailing_newline: record.fetch(:trailing_newline),

        language: {
          resolved: record[:language],
          pieces_analysis: record[:raw_analysis_language],
          pieces_classification: record[:raw_classification_language]
        },

        timestamps: {
          created: record.fetch(:created_at),
          updated: record.fetch(:updated_at)
        },

        markdown_metadata: {
          tags: record.fetch(:tags),
          source_url: record[:source_url],
          explanation: record[:explanation]
        },

        annotations: Array(asset.dig("annotations", "iterable")),
        tags: Array(asset.dig("tags", "iterable")),
        websites: Array(asset.dig("websites", "iterable"))
      }

      file.write(JSON.generate(normalized))
      file.write("\n")
    end
  end

  manifest_assets = []

  prepared
    .sort_by { |record| record.fetch(:filename).downcase }
    .each do |record|
      markdown = markdown_for(
        body: record.fetch(:body),
        language: record[:language],
        tags: record.fetch(:tags),
        source_url: record[:source_url],
        explanation: record[:explanation],
        created_at: record.fetch(:created_at),
        updated_at: record.fetch(:updated_at)
      )

      markdown_path =
        File.join(markdown_dir, record.fetch(:filename))

      File.binwrite(markdown_path, markdown)

      manifest_assets << {
        id: record.fetch(:id),
        original_id: record[:original_id],
        name: record.fetch(:name),
        filename: record.fetch(:filename),

        body_storage: record.fetch(:body_storage),
        body_sha256: record.fetch(:body_sha256),
        body_bytes: record.fetch(:body_bytes),
        trailing_newline: record.fetch(:trailing_newline),

        markdown_sha256:
          Digest::SHA256.hexdigest(markdown.b),

        language: record[:language],
        pieces_analysis_language:
          record[:raw_analysis_language],
        pieces_classification_language:
          record[:raw_classification_language],

        created: record.fetch(:created_at),
        updated: record.fetch(:updated_at),

        manual_tags: record.fetch(:tags),
        explanation: record[:explanation],
        source_url: record[:source_url]
      }
    end

  markdown_count =
    Dir.glob(File.join(markdown_dir, "*.md")).length

  unless markdown_count == assets.length
    raise(
      "Expected #{assets.length} Markdown files, " \
      "created #{markdown_count}"
    )
  end

  language_counts =
    prepared
      .group_by { |record| record[:language] || "unknown" }
      .transform_values(&:length)
      .sort
      .to_h

  manifest = {
    schema_version: 2,
    generated_at: Time.now.utc.iso8601,

    source: {
      path: input_path,
      sha256: raw_sha256,
      asset_count: assets.length
    },

    output: {
      markdown_count: markdown_count,
      assets_with_manual_tags:
        prepared.count { |record| record.fetch(:tags).any? },
      total_manual_tags:
        prepared.sum { |record| record.fetch(:tags).length },
      assets_with_explanation:
        prepared.count { |record| record[:explanation] },
      assets_with_source_url:
        prepared.count { |record| record[:source_url] },
      languages: language_counts
    },

    assets: manifest_assets
  }

  File.write(
    File.join(temporary_path, "manifest.json"),
    JSON.pretty_generate(manifest) + "\n"
  )

  FileUtils.rm_rf(output_path) if File.exist?(output_path)
  FileUtils.mv(temporary_path, output_path)

  puts
  puts "Pieces export complete"
  puts "----------------------"
  puts "Assets:          #{assets.length}"
  puts "Markdown files:  #{markdown_count}"
  puts "Manual tags:     #{manifest.dig(:output, :total_manual_tags)}"
  puts "Explanations:    #{manifest.dig(:output, :assets_with_explanation)}"
  puts "Source URLs:     #{manifest.dig(:output, :assets_with_source_url)}"
  puts "Raw SHA-256:     #{raw_sha256}"
  puts "Output:           #{output_path}"
  puts
  puts "Languages:"

  language_counts.each do |language, count|
    puts format("  %-16s %d", language, count)
  end
rescue
  FileUtils.rm_rf(temporary_path)
  raise
end