#!/bin/bash
# Improved download script using curl with proper User-Agent headers
# Many .gov and archive sites block bare wget

BASE="/var/www/jaeswift-homepage/unredacted/docs"
LOG="/tmp/download_v2_log.txt"
UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"

echo "=== Download V2 - $(date) ===" | tee $LOG

dl() {
    local url="$1"
    local dest="$2"
    local name="$3"
    # Skip if file already exists and is >1KB
    if [ -f "$dest" ] && [ $(stat -c%s "$dest" 2>/dev/null || echo 0) -gt 1000 ]; then
        local sz=$(echo "scale=2; $(stat -c%s "$dest")/1048576" | bc)
        echo "  [SKIP] $name - already exists (${sz}MB)" | tee -a $LOG
        return 0
    fi
    echo "[DL] $name" | tee -a $LOG
    if curl -sS -L -f --connect-timeout 30 --max-time 300 \
         -H "User-Agent: $UA" \
         -H "Accept: application/pdf,*/*" \
         -o "$dest" "$url" 2>/dev/null; then
        local size=$(stat -c%s "$dest" 2>/dev/null || echo 0)
        if [ "$size" -gt 1000 ]; then
            local sizemb=$(echo "scale=2; $size/1048576" | bc)
            echo "  [OK] ${sizemb}MB" | tee -a $LOG
            return 0
        else
            echo "  [FAIL] Too small (${size}B)" | tee -a $LOG
            rm -f "$dest"
            return 1
        fi
    else
        echo "  [FAIL] curl error" | tee -a $LOG
        rm -f "$dest"
        return 1
    fi
}

###############################################
# SECTION 1: UFO/UAP - USA
###############################################
echo "" | tee -a $LOG
echo "===== PENTAGON UAP REPORTS =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/pentagon-uap-reports"
mkdir -p "$DIR"

# ODNI Preliminary Assessment 2021
dl "https://www.dni.gov/files/ODNI/documents/assessments/Prelimary-Assessment-UAP-20210625.pdf" \
   "$DIR/dni-preliminary-assessment-uap-2021.pdf" "ODNI Preliminary Assessment 2021"

# Alternate URL for ODNI 2021
dl "https://www.dni.gov/files/documents/FOIA/DF-2021-00275-Preliminary-Assessment-Unidentified-Aerial-Phenomena.pdf" \
   "$DIR/dni-preliminary-assessment-uap-2021-foia.pdf" "ODNI 2021 FOIA Version"

# ODNI Annual Report 2022
dl "https://www.dni.gov/files/ODNI/documents/assessments/Unclassified-2022-Annual-Report-UAP.pdf" \
   "$DIR/dni-annual-report-uap-2022.pdf" "ODNI Annual Report 2022"

# AARO Historical Record Vol 1 (March 2024)
dl "https://media.defense.gov/2024/Mar/08/2003409233/-1/-1/0/DOPSR-CLEARED-508-COMPLIANT-HRRV1-08-MAR-2024-FINAL.PDF" \
   "$DIR/aaro-historical-record-vol1-2024.pdf" "AARO Historical Record Vol 1"

# FY2024 Consolidated Annual Report
dl "https://media.defense.gov/2024/Nov/14/2003583603/-1/-1/0/FY24-CONSOLIDATED-ANNUAL-REPORT-ON-UAP-508.PDF" \
   "$DIR/fy2024-consolidated-annual-report-uap.pdf" "FY2024 UAP Report"

# DoD IG Evaluation
dl "https://media.defense.gov/2024/Jan/25/2003381266/-1/-1/1/UNCLASSIFIED%20SUMMARY_UNIDENTIFIED%20ANOMALOUS%20PHENOMENA%20SECURE.PDF" \
   "$DIR/dodig-2023-109-uap-evaluation.pdf" "DoD IG UAP Evaluation"

# Deputy SecDef Memo
dl "https://media.defense.gov/2021/Jun/25/2002749916/-1/-1/1/DEPUTY-SECRETARY-OF-DEFENSE-MEMORANDUM-ON-UNIDENTIFIED-AERIAL-PHENOMENA-ASSESSMENTS.PDF" \
   "$DIR/depsecdef-memo-uap-2021.pdf" "DepSecDef UAP Memo 2021"

# AARO Nov 2024 consolidated
dl "https://www.dni.gov/files/ODNI/documents/assessments/DOD-AARO-Consolidated-Annual-Report-on-UAP-Nov2024.pdf" \
   "$DIR/aaro-consolidated-nov2024.pdf" "AARO Consolidated Nov 2024"

echo "" | tee -a $LOG
echo "===== PROJECT BLUE BOOK =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/project-blue-book"
mkdir -p "$DIR"

# Blue Book from Black Vault (this one worked before at 0.57MB)
# Try more Blue Book sources
dl "https://documents.theblackvault.com/documents/ufos/bluebook/bluebook-unknowns.pdf" \
   "$DIR/blue-book-unknowns.pdf" "Blue Book Unknowns"
dl "https://documents.theblackvault.com/documents/ufos/bluebook/bb-specialreport14.pdf" \
   "$DIR/blue-book-special-report-14.pdf" "Blue Book Special Report 14"
dl "https://media.defense.gov/2011/Jul/19/2001713839/-1/-1/0/AFD-101027-030.pdf" \
   "$DIR/blue-book-special-report-14-usaf.pdf" "Blue Book SR14 USAF"

echo "" | tee -a $LOG
echo "===== CIA UFO COLLECTION =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/cia-ufo-collection"
mkdir -p "$DIR"

# CIA Reading Room UFO docs (known stable URLs)
dl "https://www.cia.gov/readingroom/docs/DOC_0000015467.pdf" \
   "$DIR/cia-ufo-doc-0000015467.pdf" "CIA UFO Doc 15467"
dl "https://www.cia.gov/readingroom/docs/DOC_0000015468.pdf" \
   "$DIR/cia-ufo-doc-0000015468.pdf" "CIA UFO Doc 15468"
dl "https://www.cia.gov/readingroom/docs/DOC_0000042353.pdf" \
   "$DIR/cia-ufo-doc-0000042353.pdf" "CIA UFO Doc 42353"
dl "https://www.cia.gov/readingroom/docs/DOC_0005516020.pdf" \
   "$DIR/cia-ufo-doc-0005516020.pdf" "CIA UFO Doc 5516020"
dl "https://www.cia.gov/readingroom/docs/DOC_0005517742.pdf" \
   "$DIR/cia-ufo-doc-0005517742.pdf" "CIA UFO Doc 5517742"
dl "https://www.cia.gov/readingroom/docs/DOC_0005517761.pdf" \
   "$DIR/cia-ufo-doc-0005517761.pdf" "CIA UFO Doc 5517761"

echo "" | tee -a $LOG
echo "===== NSA UFO DOCUMENTS =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/nsa-ufo-documents"
mkdir -p "$DIR"

# NSA FOIA UFO docs (nsa.gov)
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/ufo_hypothesis.pdf" \
   "$DIR/nsa-ufo-hypothesis.pdf" "NSA UFO Hypothesis"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/in_camera_affidavit.pdf" \
   "$DIR/nsa-in-camera-affidavit.pdf" "NSA In Camera Affidavit"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/key_to_extraterrestrial_messages.pdf" \
   "$DIR/nsa-key-to-et-messages.pdf" "NSA Key to ET Messages"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/yeoman_1st_class.pdf" \
   "$DIR/nsa-yeoman-report.pdf" "NSA Yeoman Report"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/now_you_see.pdf" \
   "$DIR/nsa-now-you-see.pdf" "NSA Now You See"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/report_unidentified_flying_objects.pdf" \
   "$DIR/nsa-report-ufos.pdf" "NSA Report UFOs"

###############################################
# SECTION 1b: UFO/UAP - International
###############################################
echo "" | tee -a $LOG
echo "===== FRANCE: COMETA REPORT =====" | tee -a $LOG
DIR="$BASE/ufo-uap/FR/cometa-report"
mkdir -p "$DIR"

dl "https://web.archive.org/web/2023/https://www.ufoevidence.org/newsite/files/COMETA-part1.pdf" \
   "$DIR/cometa-report-part1.pdf" "COMETA Part 1 (Archive)"
dl "https://web.archive.org/web/2023/https://www.ufoevidence.org/newsite/files/COMETA-part2.pdf" \
   "$DIR/cometa-report-part2.pdf" "COMETA Part 2 (Archive)"

echo "" | tee -a $LOG
echo "===== BRAZIL: OPERATION SAUCER =====" | tee -a $LOG
DIR="$BASE/ufo-uap/BR/operation-saucer"
mkdir -p "$DIR"

# Try Wayback Machine for Black Vault Brazil docs
dl "https://web.archive.org/web/2024/https://documents.theblackvault.com/documents/ufos/brazil/brazil-ufo-night-official-document.pdf" \
   "$DIR/brazil-ufo-official.pdf" "Brazil UFO Official"

echo "" | tee -a $LOG
echo "===== AUSTRALIA: RAAF FILES =====" | tee -a $LOG
DIR="$BASE/ufo-uap/AU/raaf-ufo-files"
mkdir -p "$DIR"

# NAA Australia UFO files
dl "https://recordsearch.naa.gov.au/SearchNRetrieve/NAAMedia/ShowImage.aspx?B=30030606&S=1" \
   "$DIR/naa-30030606.pdf" "NAA Record 30030606"

echo "" | tee -a $LOG
echo "===== CANADA: PROJECT MAGNET =====" | tee -a $LOG
DIR="$BASE/ufo-uap/CA/project-magnet"
mkdir -p "$DIR"

# Project Magnet - Try Wayback
dl "https://web.archive.org/web/2024/https://documents.theblackvault.com/documents/ufos/canada/project-magnet.pdf" \
   "$DIR/project-magnet.pdf" "Project Magnet"

echo "" | tee -a $LOG
echo "===== NEW ZEALAND: NZDF FILES =====" | tee -a $LOG
DIR="$BASE/ufo-uap/NZ/nzdf-ufo-files"
mkdir -p "$DIR"

# NZDF released files
dl "https://www.nzdf.mil.nz/assets/Downloads/NZDF-UFO-Files.pdf" \
   "$DIR/nzdf-ufo-files.pdf" "NZDF UFO Files"

###############################################
# SECTION 2: COVERT OPERATIONS
###############################################
echo "" | tee -a $LOG
echo "===== MKULTRA =====" | tee -a $LOG
DIR="$BASE/covert-operations/mkultra"
mkdir -p "$DIR"

dl "https://www.cia.gov/readingroom/docs/DOC_0000017468.pdf" \
   "$DIR/mkultra-doc-0000017468.pdf" "MKUltra Doc 17468"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP88-01315R000300110021-8.pdf" \
   "$DIR/mkultra-cia-rdp88.pdf" "MKUltra CIA RDP88"
dl "https://www.intelligence.senate.gov/sites/default/files/hearings/95mkultra.pdf" \
   "$DIR/mkultra-senate-hearing-1977.pdf" "MKUltra Senate Hearing 1977"

echo "" | tee -a $LOG
echo "===== COINTELPRO =====" | tee -a $LOG
DIR="$BASE/covert-operations/cointelpro"
mkdir -p "$DIR"

dl "https://vault.fbi.gov/cointel-pro/cointel-pro-black-extremist/cointelpro-black-extremists-part-01-of/at_download/file" \
   "$DIR/cointelpro-black-extremists-part1.pdf" "COINTELPRO Black Extremists Pt1"
dl "https://vault.fbi.gov/cointel-pro/cointel-pro-white-hate/cointelpro-white-hate-groups-part-01-of/at_download/file" \
   "$DIR/cointelpro-white-hate-part1.pdf" "COINTELPRO White Hate Pt1"
dl "https://vault.fbi.gov/cointel-pro/new-left/cointelpro-new-left-part-01-of/at_download/file" \
   "$DIR/cointelpro-new-left-part1.pdf" "COINTELPRO New Left Pt1"

echo "" | tee -a $LOG
echo "===== STARGATE PROGRAM =====" | tee -a $LOG
DIR="$BASE/covert-operations/stargate-program"
mkdir -p "$DIR"

dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00788R001700210016-5.pdf" \
   "$DIR/stargate-remote-viewing-session-1.pdf" "Stargate RV Session 1"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00789R003100030001-4.pdf" \
   "$DIR/stargate-remote-viewing-session-2.pdf" "Stargate RV Session 2"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00787R000500250001-0.pdf" \
   "$DIR/stargate-project-sun-streak.pdf" "Stargate Sun Streak"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00788R001900680001-4.pdf" \
   "$DIR/stargate-evaluation.pdf" "Stargate Evaluation"

echo "" | tee -a $LOG
echo "===== OPERATION NORTHWOODS =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-northwoods"
mkdir -p "$DIR"

dl "https://nsarchive2.gwu.edu/news/20010430/northwoods.pdf" \
   "$DIR/operation-northwoods.pdf" "Operation Northwoods"
# Backup from archive.org
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/news/20010430/northwoods.pdf" \
   "$DIR/operation-northwoods.pdf" "Operation Northwoods (Archive)"

echo "" | tee -a $LOG
echo "===== OPERATION MOCKINGBIRD =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-mockingbird"
mkdir -p "$DIR"

# Church Committee Report on CIA media manipulation
dl "https://www.intelligence.senate.gov/sites/default/files/94755_I.pdf" \
   "$DIR/church-committee-book1.pdf" "Church Committee Book I"
dl "https://www.intelligence.senate.gov/sites/default/files/94755_II.pdf" \
   "$DIR/church-committee-book2.pdf" "Church Committee Book II"
dl "https://www.intelligence.senate.gov/sites/default/files/94755_III.pdf" \
   "$DIR/church-committee-book3.pdf" "Church Committee Book III"

echo "" | tee -a $LOG
echo "===== OPERATION PAPERCLIP =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-paperclip"
mkdir -p "$DIR"

dl "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB146/doc1.pdf" \
   "$DIR/paperclip-jioa-memo.pdf" "Paperclip JIOA Memo"
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/NSAEBB/NSAEBB146/doc1.pdf" \
   "$DIR/paperclip-jioa-memo.pdf" "Paperclip JIOA (Archive)"

###############################################
# SECTION 3: GOVERNMENT
###############################################
echo "" | tee -a $LOG
echo "===== JFK ASSASSINATION =====" | tee -a $LOG
DIR="$BASE/government/jfk-assassination"
mkdir -p "$DIR"

# Warren Commission
dl "https://www.archives.gov/research/jfk/warren-commission-report/warren-commission-report.pdf" \
   "$DIR/warren-commission-report.pdf" "Warren Commission Report"
# Try history-matters for Warren Commission
dl "https://history-matters.com/archive/jfk/wc/wr/pdf/WC_Vol0.pdf" \
   "$DIR/warren-commission-vol0.pdf" "Warren Commission Vol 0"
dl "https://www.maryferrell.org/showDoc.html?docId=946" \
   "$DIR/jfk-mary-ferrell.pdf" "JFK Mary Ferrell"

echo "" | tee -a $LOG
echo "===== PENTAGON PAPERS =====" | tee -a $LOG
DIR="$BASE/government/pentagon-papers"
mkdir -p "$DIR"

dl "https://www.archives.gov/research/pentagon-papers" \
   "$DIR/pentagon-papers-nara.pdf" "Pentagon Papers NARA"
dl "https://media.nara.gov/research/pentagon-papers/Pentagon-Papers-Part-I.pdf" \
   "$DIR/pentagon-papers-part1.pdf" "Pentagon Papers Part I"

echo "" | tee -a $LOG
echo "===== WATERGATE =====" | tee -a $LOG
DIR="$BASE/government/watergate"
mkdir -p "$DIR"

dl "https://vault.fbi.gov/watergate/watergate-part-01-of/at_download/file" \
   "$DIR/watergate-fbi-part1.pdf" "Watergate FBI Part 1"

echo "" | tee -a $LOG
echo "===== IRAN-CONTRA =====" | tee -a $LOG
DIR="$BASE/government/iran-contra"
mkdir -p "$DIR"

dl "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB210/iran-contra-guide.pdf" \
   "$DIR/iran-contra-guide.pdf" "Iran-Contra Guide"
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/NSAEBB/NSAEBB210/iran-contra-guide.pdf" \
   "$DIR/iran-contra-guide.pdf" "Iran-Contra Guide (Archive)"

echo "" | tee -a $LOG
echo "===== CIA TORTURE REPORT =====" | tee -a $LOG
DIR="$BASE/government/cia-torture-report"
mkdir -p "$DIR"

# SSCI Torture Report Executive Summary (525 pages)
dl "https://www.intelligence.senate.gov/sites/default/files/publications/CRPT-113srpt288.pdf" \
   "$DIR/ssci-torture-report-executive-summary.pdf" "SSCI Torture Report"
# Feinstein CIA study
dl "https://www.feinstein.senate.gov/public/_cache/files/7/c/7c85429a-ec38-4bb5-968f-289799bf6d0e/D87288C34A6D9FF736F9459ABCF83210.sscistudy1.pdf" \
   "$DIR/feinstein-cia-study.pdf" "Feinstein CIA Study"


###############################################
# SUMMARY
###############################################
echo "" | tee -a $LOG
echo "========== DOWNLOAD V2 SUMMARY ==========" | tee -a $LOG
echo "" | tee -a $LOG

for dir in $(find $BASE -type d | sort); do
    count=$(find "$dir" -maxdepth 1 -type f \( -name '*.pdf' -o -name '*.PDF' \) | wc -l)
    if [ "$count" -gt 0 ]; then
        size=$(du -sh "$dir" 2>/dev/null | cut -f1)
        echo "  $dir: $count files ($size)" | tee -a $LOG
    fi
done

echo "" | tee -a $LOG
# Include UK existing docs in total
TOTAL_FILES=$(find $BASE -type f \( -name '*.pdf' -o -name '*.PDF' \) | wc -l)
TOTAL_SIZE=$(du -sh $BASE 2>/dev/null | cut -f1)
echo "TOTAL: $TOTAL_FILES PDF files, $TOTAL_SIZE total" | tee -a $LOG
echo "=== Download V2 complete - $(date) ===" | tee -a $LOG