#!/bin/bash
# Master download script for UNREDACTED document collections
# Run directly on VPS: bash /tmp/download_all_docs.sh

# set -e  # disabled - many downloads expected to fail
BASE="/var/www/jaeswift-homepage/unredacted/docs"
LOG="/tmp/download_log.txt"
echo "=== UNREDACTED Document Download - $(date) ===" | tee $LOG

download() {
    local url="$1"
    local dest="$2"
    local name="$3"
    echo "[DL] $name" | tee -a $LOG
    if wget -q --timeout=60 --tries=3 -O "$dest" "$url" 2>/dev/null; then
        local size=$(stat -c%s "$dest" 2>/dev/null || echo 0)
        if [ "$size" -gt 1000 ]; then
            local sizemb=$(echo "scale=2; $size/1048576" | bc)
            echo "  [OK] ${sizemb}MB - $dest" | tee -a $LOG
        else
            echo "  [FAIL] Too small (${size}B), removing" | tee -a $LOG
            rm -f "$dest"
        fi
    else
        echo "  [FAIL] wget failed" | tee -a $LOG
        rm -f "$dest"
    fi
}

#############################################
# SECTION 1: UFO/UAP Collections
#############################################
echo "" | tee -a $LOG
echo "========== UFO/UAP COLLECTIONS ==========" | tee -a $LOG

# --- USA: Pentagon UAP Reports ---
DIR="$BASE/ufo-uap/USA/pentagon-uap-reports"
mkdir -p "$DIR"
echo "--- Pentagon UAP Reports ---" | tee -a $LOG
download "https://www.dni.gov/files/ODNI/documents/assessments/Prelimary-Assessment-UAP-20210625.pdf" "$DIR/dni-preliminary-assessment-uap-2021.pdf" "ODNI Preliminary Assessment 2021"
download "https://www.dni.gov/files/ODNI/documents/assessments/Unclassified-2022-Annual-Report-UAP.pdf" "$DIR/dni-annual-report-uap-2022.pdf" "ODNI Annual Report UAP 2022"
download "https://media.defense.gov/2024/Mar/08/2003409233/-1/-1/0/DOPSR-CLEARED-508-COMPLIANT-HRRV1-08-MAR-2024-FINAL.PDF" "$DIR/aaro-historical-record-report-vol1-2024.pdf" "AARO Historical Record Vol 1"
download "https://media.defense.gov/2024/Nov/14/2003583603/-1/-1/0/FY24-CONSOLIDATED-ANNUAL-REPORT-ON-UAP-508.PDF" "$DIR/fy2024-consolidated-annual-report-uap.pdf" "FY2024 UAP Annual Report"
download "https://media.defense.gov/2024/Jan/25/2003381266/-1/-1/1/UNCLASSIFIED%20SUMMARY_UNIDENTIFIED%20ANOMALOUS%20PHENOMENA%20SECURE.PDF" "$DIR/dodig-2023-109-uap-evaluation.pdf" "DoD IG UAP Evaluation 2023"
download "https://www.dni.gov/files/ODNI/documents/assessments/DOD-AARO-Consolidated-Annual-Report-on-UAP-Nov2024.pdf" "$DIR/aaro-consolidated-annual-report-uap-nov2024.pdf" "AARO Consolidated Nov 2024"
download "https://media.defense.gov/2021/Jun/25/2002749916/-1/-1/1/DEPUTY-SECRETARY-OF-DEFENSE-MEMORANDUM-ON-UNIDENTIFIED-AERIAL-PHENOMENA-ASSESSMENTS.PDF" "$DIR/depsecdef-memo-uap-assessments-2021.pdf" "DepSecDef UAP Memo 2021"

# --- USA: Project Blue Book ---
DIR="$BASE/ufo-uap/USA/project-blue-book"
mkdir -p "$DIR"
echo "--- Project Blue Book ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/fbifiles/bluebook.pdf" "$DIR/project-blue-book-fbi.pdf" "Blue Book FBI File"
download "https://documents.theblackvault.com/documents/ufos/bluebook/bluebook.pdf" "$DIR/project-blue-book-archive.pdf" "Blue Book Archive"
# Try individual Blue Book case files from known Black Vault paths
download "https://documents.theblackvault.com/documents/ufos/bluebook/ProjectBlueBookSpecialReport14.pdf" "$DIR/project-blue-book-special-report-14.pdf" "Blue Book Special Report 14"
download "https://documents2.theblackvault.com/documents/ufos/bluebook/BlueBookIndexByState.pdf" "$DIR/blue-book-index-by-state.pdf" "Blue Book Index by State"

# --- USA: CIA UFO Collection ---
DIR="$BASE/ufo-uap/USA/cia-ufo-collection"
mkdir -p "$DIR"
echo "--- CIA UFO Collection ---" | tee -a $LOG
download "https://documents2.theblackvault.com/documents/ufos/CIA-UFO-CollectionCDROM.zip" "$DIR/CIA-UFO-CollectionCDROM.zip" "CIA UFO CD-ROM Collection"
# If zip downloaded, extract it
if [ -f "$DIR/CIA-UFO-CollectionCDROM.zip" ]; then
    echo "  Extracting CIA UFO zip..." | tee -a $LOG
    cd "$DIR" && unzip -qo CIA-UFO-CollectionCDROM.zip 2>/dev/null && rm -f CIA-UFO-CollectionCDROM.zip
    echo "  [OK] Extracted" | tee -a $LOG
    cd /
fi

# --- USA: NSA UFO Documents ---
DIR="$BASE/ufo-uap/USA/nsa-ufo-documents"
mkdir -p "$DIR"
echo "--- NSA UFO Documents ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/nsa/ufo/nsa-ufo-documents.pdf" "$DIR/nsa-ufo-documents.pdf" "NSA UFO Documents"
download "https://documents2.theblackvault.com/documents/nsa/nsa-ufo.pdf" "$DIR/nsa-ufo-collection.pdf" "NSA UFO Collection"
# Try NSA FOIA released UFO docs
download "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/in_camera_affidavit.pdf" "$DIR/nsa-in-camera-affidavit.pdf" "NSA In Camera Affidavit"
download "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/ufo_hypothesis.pdf" "$DIR/nsa-ufo-hypothesis.pdf" "NSA UFO Hypothesis"
download "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/key_to_extraterrestrial_messages.pdf" "$DIR/nsa-key-to-et-messages.pdf" "NSA Key to ET Messages"

# --- France: COMETA Report ---
DIR="$BASE/ufo-uap/FR/cometa-report"
mkdir -p "$DIR"
echo "--- COMETA Report ---" | tee -a $LOG
download "https://www.ufoevidence.org/newsite/files/COMETA-part1.pdf" "$DIR/cometa-report-part1.pdf" "COMETA Part 1"
download "https://www.ufoevidence.org/newsite/files/COMETA-part2.pdf" "$DIR/cometa-report-part2.pdf" "COMETA Part 2"
download "https://documents.theblackvault.com/documents/ufos/cometa-report.pdf" "$DIR/cometa-report-full.pdf" "COMETA Full Report"

# --- Brazil: Operation Saucer ---
DIR="$BASE/ufo-uap/BR/operation-saucer"
mkdir -p "$DIR"
echo "--- Operation Saucer / Operação Prato ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/ufos/brazil/brazil-ufo-files.pdf" "$DIR/brazil-ufo-files.pdf" "Brazil UFO Files"
download "https://documents.theblackvault.com/documents/ufos/brazil/operacao-prato.pdf" "$DIR/operacao-prato.pdf" "Operação Prato"

# --- Australia: RAAF UFO Files ---
DIR="$BASE/ufo-uap/AU/raaf-ufo-files"
mkdir -p "$DIR"
echo "--- Australian RAAF UFO Files ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/ufos/australia/raaf-ufo-files.pdf" "$DIR/raaf-ufo-files.pdf" "RAAF UFO Files"
download "https://recordsearch.naa.gov.au/SearchNRetrieve/NAAMedia/ShowImage.aspx?B=30030606" "$DIR/naa-ufo-30030606.pdf" "NAA UFO Record 30030606"

# --- Canada: Project Magnet ---
DIR="$BASE/ufo-uap/CA/project-magnet"
mkdir -p "$DIR"
echo "--- Canadian Project Magnet ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/ufos/canada/project-magnet.pdf" "$DIR/project-magnet.pdf" "Project Magnet"
download "https://documents.theblackvault.com/documents/ufos/canada/canadian-ufo-files.pdf" "$DIR/canadian-ufo-files.pdf" "Canadian UFO Files"

# --- New Zealand: NZDF UFO Files ---
DIR="$BASE/ufo-uap/NZ/nzdf-ufo-files"
mkdir -p "$DIR"
echo "--- New Zealand NZDF UFO Files ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/ufos/newzealand/nzdf-ufo-files.pdf" "$DIR/nzdf-ufo-files.pdf" "NZDF UFO Files"


#############################################
# SECTION 2: Covert Operations
#############################################
echo "" | tee -a $LOG
echo "========== COVERT OPERATIONS ==========" | tee -a $LOG

# --- MKUltra ---
DIR="$BASE/covert-operations/mkultra"
mkdir -p "$DIR"
echo "--- MKUltra ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/mindcontrol/mkultra-cia.pdf" "$DIR/mkultra-cia.pdf" "MKUltra CIA"
download "https://documents.theblackvault.com/documents/mindcontrol/mkultra.pdf" "$DIR/mkultra-documents.pdf" "MKUltra Documents"
download "https://documents.theblackvault.com/documents/mindcontrol/mkultra-briefing.pdf" "$DIR/mkultra-briefing.pdf" "MKUltra Briefing"
# Try CIA reading room path
download "https://www.cia.gov/readingroom/docs/CIA-RDP88-01315R000300110021-8.pdf" "$DIR/mkultra-declassified-1.pdf" "MKUltra Declassified 1"

# --- COINTELPRO ---
DIR="$BASE/covert-operations/cointelpro"
mkdir -p "$DIR"
echo "--- COINTELPRO ---" | tee -a $LOG
download "https://vault.fbi.gov/cointel-pro/cointel-pro-black-extremist/cointelpro-black-extremists-part-01-of/view" "$DIR/cointelpro-black-extremists-part1.pdf" "COINTELPRO Black Extremists Part 1"
download "https://documents.theblackvault.com/documents/fbifiles/cointelpro.pdf" "$DIR/cointelpro-fbi.pdf" "COINTELPRO FBI"
download "https://documents.theblackvault.com/documents/fbifiles/cointelpro-new-left.pdf" "$DIR/cointelpro-new-left.pdf" "COINTELPRO New Left"

# --- Stargate Program ---
DIR="$BASE/covert-operations/stargate-program"
mkdir -p "$DIR"
echo "--- Stargate Program ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/cia/stargate/stargate.pdf" "$DIR/stargate-cia.pdf" "Stargate CIA"
download "https://www.cia.gov/readingroom/docs/CIA-RDP96-00788R001700210016-5.pdf" "$DIR/stargate-remote-viewing-1.pdf" "Stargate Remote Viewing 1"
download "https://www.cia.gov/readingroom/docs/CIA-RDP96-00789R003100030001-4.pdf" "$DIR/stargate-remote-viewing-2.pdf" "Stargate Remote Viewing 2"
download "https://www.cia.gov/readingroom/docs/CIA-RDP96-00787R000500250001-0.pdf" "$DIR/stargate-project-overview.pdf" "Stargate Project Overview"

# --- Operation Northwoods ---
DIR="$BASE/covert-operations/operation-northwoods"
mkdir -p "$DIR"
echo "--- Operation Northwoods ---" | tee -a $LOG
download "https://nsarchive2.gwu.edu/news/20010430/northwoods.pdf" "$DIR/operation-northwoods.pdf" "Operation Northwoods Memo"
download "https://documents.theblackvault.com/documents/dod/operation-northwoods.pdf" "$DIR/operation-northwoods-blackvault.pdf" "Operation Northwoods (BV)"

# --- Operation Mockingbird ---
DIR="$BASE/covert-operations/operation-mockingbird"
mkdir -p "$DIR"
echo "--- Operation Mockingbird ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/cia/operation-mockingbird.pdf" "$DIR/operation-mockingbird.pdf" "Operation Mockingbird"
download "https://www.cia.gov/readingroom/docs/CIA-RDP88-01315R000300110021-8.pdf" "$DIR/cia-media-operations.pdf" "CIA Media Operations"

# --- Operation Paperclip ---
DIR="$BASE/covert-operations/operation-paperclip"
mkdir -p "$DIR"
echo "--- Operation Paperclip ---" | tee -a $LOG
download "https://documents.theblackvault.com/documents/dod/operation-paperclip.pdf" "$DIR/operation-paperclip.pdf" "Operation Paperclip"
download "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB146/doc1.pdf" "$DIR/paperclip-jioa-memo.pdf" "Paperclip JIOA Memo"


#############################################
# SECTION 3: Government Collections
#############################################
echo "" | tee -a $LOG
echo "========== GOVERNMENT COLLECTIONS ==========" | tee -a $LOG

# --- JFK Assassination ---
DIR="$BASE/government/jfk-assassination"
mkdir -p "$DIR"
echo "--- JFK Assassination ---" | tee -a $LOG
download "https://www.archives.gov/files/research/jfk/releases/2023/0035-jfk-overview.pdf" "$DIR/jfk-overview-2023.pdf" "JFK Overview 2023"
download "https://www.history-matters.com/archive/jfk/wc/wr/pdf/WC_Vol0.pdf" "$DIR/warren-commission-report.pdf" "Warren Commission Report"
download "https://documents.theblackvault.com/documents/jfk/jfk-assassination-files.pdf" "$DIR/jfk-assassination-files.pdf" "JFK Assassination Files"

# --- Pentagon Papers ---
DIR="$BASE/government/pentagon-papers"
mkdir -p "$DIR"
echo "--- Pentagon Papers ---" | tee -a $LOG
download "https://www.archives.gov/files/research/pentagon-papers/pentagon-papers-overview.pdf" "$DIR/pentagon-papers-overview.pdf" "Pentagon Papers Overview"
download "https://documents.theblackvault.com/documents/pentagon-papers/pentagon-papers.pdf" "$DIR/pentagon-papers.pdf" "Pentagon Papers"

# --- Watergate ---
DIR="$BASE/government/watergate"
mkdir -p "$DIR"
echo "--- Watergate ---" | tee -a $LOG
download "https://vault.fbi.gov/watergate/watergate-part-01-of/view" "$DIR/watergate-fbi-part1.pdf" "Watergate FBI Part 1"
download "https://documents.theblackvault.com/documents/fbifiles/watergate.pdf" "$DIR/watergate-fbi.pdf" "Watergate FBI Files"

# --- Iran-Contra ---
DIR="$BASE/government/iran-contra"
mkdir -p "$DIR"
echo "--- Iran-Contra ---" | tee -a $LOG
download "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB210/iran-contra-guide.pdf" "$DIR/iran-contra-guide.pdf" "Iran-Contra Guide"
download "https://documents.theblackvault.com/documents/cia/iran-contra.pdf" "$DIR/iran-contra-cia.pdf" "Iran-Contra CIA"

# --- CIA Torture Report ---
DIR="$BASE/government/cia-torture-report"
mkdir -p "$DIR"
echo "--- CIA Torture Report (SSCI) ---" | tee -a $LOG
download "https://www.intelligence.senate.gov/sites/default/files/publications/CRPT-113srpt288.pdf" "$DIR/ssci-cia-torture-report-executive-summary.pdf" "SSCI Torture Report Executive Summary"
download "https://documents.theblackvault.com/documents/cia/cia-torture-report.pdf" "$DIR/cia-torture-report.pdf" "CIA Torture Report"


#############################################
# SUMMARY
#############################################
echo "" | tee -a $LOG
echo "========== DOWNLOAD SUMMARY ==========" | tee -a $LOG
echo "" | tee -a $LOG

# Count files per directory
for dir in $(find $BASE -type d | sort); do
    count=$(find "$dir" -maxdepth 1 -type f -name '*.pdf' | wc -l)
    if [ "$count" -gt 0 ]; then
        size=$(du -sh "$dir" 2>/dev/null | cut -f1)
        echo "  $dir: $count files ($size)" | tee -a $LOG
    fi
done

echo "" | tee -a $LOG
TOTAL_FILES=$(find $BASE -type f -name '*.pdf' | wc -l)
TOTAL_SIZE=$(du -sh $BASE 2>/dev/null | cut -f1)
echo "TOTAL: $TOTAL_FILES PDF files, $TOTAL_SIZE" | tee -a $LOG
echo "=== Download complete - $(date) ===" | tee -a $LOG