jae/jaeswift-website
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
jaeswift-website/download_v2.sh
jae 054cf547cc feat: add UNREDACTED categories (US UFO, Covert Ops, Government) + CRIME SCENE section 
- Added 3 categories to unredacted.json: UFO/UAP (US), Covert Operations, Government
- 127 real declassified PDFs (4.8GB) on VPS including:
  - UK MOD UFO files (112 files, 4.3GB)
  - Project Condign (250MB)
  - Pentagon UAP Report (2021)
  - CIA/NSA UFO documents
  - MKUltra Senate Hearing
  - Stargate Project docs
  - Operation Northwoods/Paperclip
  - Pentagon Papers (62MB)
  - CIA Torture Report (44MB)
  - JFK/Iran-Contra docs
- New CRIME SCENE section: HTML, CSS, JS, JSON, API routes
- Added to navigation and depot landing page
2026-04-15 19:41:09 +00:00

337 lines
14 KiB
Bash
Raw Blame History

#!/bin/bash
# Improved download script using curl with proper User-Agent headers
# Many .gov and archive sites block bare wget
BASE="/var/www/jaeswift-homepage/unredacted/docs"
LOG="/tmp/download_v2_log.txt"
UA="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
echo "=== Download V2 - $(date) ===" | tee $LOG
dl() {
local url="$1"
local dest="$2"
local name="$3"
# Skip if file already exists and is >1KB
if [ -f "$dest" ] && [ $(stat -c%s "$dest" 2>/dev/null || echo 0) -gt 1000 ]; then
local sz=$(echo "scale=2; $(stat -c%s "$dest")/1048576" | bc)
echo " [SKIP] $name - already exists (${sz}MB)" | tee -a $LOG
return 0
fi
echo "[DL] $name" | tee -a $LOG
if curl -sS -L -f --connect-timeout 30 --max-time 300 \
-H "User-Agent: $UA" \
-H "Accept: application/pdf,*/*" \
-o "$dest" "$url" 2>/dev/null; then
local size=$(stat -c%s "$dest" 2>/dev/null || echo 0)
if [ "$size" -gt 1000 ]; then
local sizemb=$(echo "scale=2; $size/1048576" | bc)
echo " [OK] ${sizemb}MB" | tee -a $LOG
return 0
else
echo " [FAIL] Too small (${size}B)" | tee -a $LOG
rm -f "$dest"
return 1
fi
else
echo " [FAIL] curl error" | tee -a $LOG
rm -f "$dest"
return 1
fi
}
###############################################
# SECTION 1: UFO/UAP - USA
###############################################
echo "" | tee -a $LOG
echo "===== PENTAGON UAP REPORTS =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/pentagon-uap-reports"
mkdir -p "$DIR"
# ODNI Preliminary Assessment 2021
dl "https://www.dni.gov/files/ODNI/documents/assessments/Prelimary-Assessment-UAP-20210625.pdf" \
"$DIR/dni-preliminary-assessment-uap-2021.pdf" "ODNI Preliminary Assessment 2021"
# Alternate URL for ODNI 2021
dl "https://www.dni.gov/files/documents/FOIA/DF-2021-00275-Preliminary-Assessment-Unidentified-Aerial-Phenomena.pdf" \
"$DIR/dni-preliminary-assessment-uap-2021-foia.pdf" "ODNI 2021 FOIA Version"
# ODNI Annual Report 2022
dl "https://www.dni.gov/files/ODNI/documents/assessments/Unclassified-2022-Annual-Report-UAP.pdf" \
"$DIR/dni-annual-report-uap-2022.pdf" "ODNI Annual Report 2022"
# AARO Historical Record Vol 1 (March 2024)
dl "https://media.defense.gov/2024/Mar/08/2003409233/-1/-1/0/DOPSR-CLEARED-508-COMPLIANT-HRRV1-08-MAR-2024-FINAL.PDF" \
"$DIR/aaro-historical-record-vol1-2024.pdf" "AARO Historical Record Vol 1"
# FY2024 Consolidated Annual Report
dl "https://media.defense.gov/2024/Nov/14/2003583603/-1/-1/0/FY24-CONSOLIDATED-ANNUAL-REPORT-ON-UAP-508.PDF" \
"$DIR/fy2024-consolidated-annual-report-uap.pdf" "FY2024 UAP Report"
# DoD IG Evaluation
dl "https://media.defense.gov/2024/Jan/25/2003381266/-1/-1/1/UNCLASSIFIED%20SUMMARY_UNIDENTIFIED%20ANOMALOUS%20PHENOMENA%20SECURE.PDF" \
"$DIR/dodig-2023-109-uap-evaluation.pdf" "DoD IG UAP Evaluation"
# Deputy SecDef Memo
dl "https://media.defense.gov/2021/Jun/25/2002749916/-1/-1/1/DEPUTY-SECRETARY-OF-DEFENSE-MEMORANDUM-ON-UNIDENTIFIED-AERIAL-PHENOMENA-ASSESSMENTS.PDF" \
"$DIR/depsecdef-memo-uap-2021.pdf" "DepSecDef UAP Memo 2021"
# AARO Nov 2024 consolidated
dl "https://www.dni.gov/files/ODNI/documents/assessments/DOD-AARO-Consolidated-Annual-Report-on-UAP-Nov2024.pdf" \
"$DIR/aaro-consolidated-nov2024.pdf" "AARO Consolidated Nov 2024"
echo "" | tee -a $LOG
echo "===== PROJECT BLUE BOOK =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/project-blue-book"
mkdir -p "$DIR"
# Blue Book from Black Vault (this one worked before at 0.57MB)
# Try more Blue Book sources
dl "https://documents.theblackvault.com/documents/ufos/bluebook/bluebook-unknowns.pdf" \
"$DIR/blue-book-unknowns.pdf" "Blue Book Unknowns"
dl "https://documents.theblackvault.com/documents/ufos/bluebook/bb-specialreport14.pdf" \
"$DIR/blue-book-special-report-14.pdf" "Blue Book Special Report 14"
dl "https://media.defense.gov/2011/Jul/19/2001713839/-1/-1/0/AFD-101027-030.pdf" \
"$DIR/blue-book-special-report-14-usaf.pdf" "Blue Book SR14 USAF"
echo "" | tee -a $LOG
echo "===== CIA UFO COLLECTION =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/cia-ufo-collection"
mkdir -p "$DIR"
# CIA Reading Room UFO docs (known stable URLs)
dl "https://www.cia.gov/readingroom/docs/DOC_0000015467.pdf" \
"$DIR/cia-ufo-doc-0000015467.pdf" "CIA UFO Doc 15467"
dl "https://www.cia.gov/readingroom/docs/DOC_0000015468.pdf" \
"$DIR/cia-ufo-doc-0000015468.pdf" "CIA UFO Doc 15468"
dl "https://www.cia.gov/readingroom/docs/DOC_0000042353.pdf" \
"$DIR/cia-ufo-doc-0000042353.pdf" "CIA UFO Doc 42353"
dl "https://www.cia.gov/readingroom/docs/DOC_0005516020.pdf" \
"$DIR/cia-ufo-doc-0005516020.pdf" "CIA UFO Doc 5516020"
dl "https://www.cia.gov/readingroom/docs/DOC_0005517742.pdf" \
"$DIR/cia-ufo-doc-0005517742.pdf" "CIA UFO Doc 5517742"
dl "https://www.cia.gov/readingroom/docs/DOC_0005517761.pdf" \
"$DIR/cia-ufo-doc-0005517761.pdf" "CIA UFO Doc 5517761"
echo "" | tee -a $LOG
echo "===== NSA UFO DOCUMENTS =====" | tee -a $LOG
DIR="$BASE/ufo-uap/USA/nsa-ufo-documents"
mkdir -p "$DIR"
# NSA FOIA UFO docs (nsa.gov)
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/ufo_hypothesis.pdf" \
"$DIR/nsa-ufo-hypothesis.pdf" "NSA UFO Hypothesis"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/in_camera_affidavit.pdf" \
"$DIR/nsa-in-camera-affidavit.pdf" "NSA In Camera Affidavit"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/key_to_extraterrestrial_messages.pdf" \
"$DIR/nsa-key-to-et-messages.pdf" "NSA Key to ET Messages"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/yeoman_1st_class.pdf" \
"$DIR/nsa-yeoman-report.pdf" "NSA Yeoman Report"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/now_you_see.pdf" \
"$DIR/nsa-now-you-see.pdf" "NSA Now You See"
dl "https://www.nsa.gov/portals/75/documents/news-features/declassified-documents/ufo/report_unidentified_flying_objects.pdf" \
"$DIR/nsa-report-ufos.pdf" "NSA Report UFOs"
###############################################
# SECTION 1b: UFO/UAP - International
###############################################
echo "" | tee -a $LOG
echo "===== FRANCE: COMETA REPORT =====" | tee -a $LOG
DIR="$BASE/ufo-uap/FR/cometa-report"
mkdir -p "$DIR"
dl "https://web.archive.org/web/2023/https://www.ufoevidence.org/newsite/files/COMETA-part1.pdf" \
"$DIR/cometa-report-part1.pdf" "COMETA Part 1 (Archive)"
dl "https://web.archive.org/web/2023/https://www.ufoevidence.org/newsite/files/COMETA-part2.pdf" \
"$DIR/cometa-report-part2.pdf" "COMETA Part 2 (Archive)"
echo "" | tee -a $LOG
echo "===== BRAZIL: OPERATION SAUCER =====" | tee -a $LOG
DIR="$BASE/ufo-uap/BR/operation-saucer"
mkdir -p "$DIR"
# Try Wayback Machine for Black Vault Brazil docs
dl "https://web.archive.org/web/2024/https://documents.theblackvault.com/documents/ufos/brazil/brazil-ufo-night-official-document.pdf" \
"$DIR/brazil-ufo-official.pdf" "Brazil UFO Official"
echo "" | tee -a $LOG
echo "===== AUSTRALIA: RAAF FILES =====" | tee -a $LOG
DIR="$BASE/ufo-uap/AU/raaf-ufo-files"
mkdir -p "$DIR"
# NAA Australia UFO files
dl "https://recordsearch.naa.gov.au/SearchNRetrieve/NAAMedia/ShowImage.aspx?B=30030606&S=1" \
"$DIR/naa-30030606.pdf" "NAA Record 30030606"
echo "" | tee -a $LOG
echo "===== CANADA: PROJECT MAGNET =====" | tee -a $LOG
DIR="$BASE/ufo-uap/CA/project-magnet"
mkdir -p "$DIR"
# Project Magnet - Try Wayback
dl "https://web.archive.org/web/2024/https://documents.theblackvault.com/documents/ufos/canada/project-magnet.pdf" \
"$DIR/project-magnet.pdf" "Project Magnet"
echo "" | tee -a $LOG
echo "===== NEW ZEALAND: NZDF FILES =====" | tee -a $LOG
DIR="$BASE/ufo-uap/NZ/nzdf-ufo-files"
mkdir -p "$DIR"
# NZDF released files
dl "https://www.nzdf.mil.nz/assets/Downloads/NZDF-UFO-Files.pdf" \
"$DIR/nzdf-ufo-files.pdf" "NZDF UFO Files"
###############################################
# SECTION 2: COVERT OPERATIONS
###############################################
echo "" | tee -a $LOG
echo "===== MKULTRA =====" | tee -a $LOG
DIR="$BASE/covert-operations/mkultra"
mkdir -p "$DIR"
dl "https://www.cia.gov/readingroom/docs/DOC_0000017468.pdf" \
"$DIR/mkultra-doc-0000017468.pdf" "MKUltra Doc 17468"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP88-01315R000300110021-8.pdf" \
"$DIR/mkultra-cia-rdp88.pdf" "MKUltra CIA RDP88"
dl "https://www.intelligence.senate.gov/sites/default/files/hearings/95mkultra.pdf" \
"$DIR/mkultra-senate-hearing-1977.pdf" "MKUltra Senate Hearing 1977"
echo "" | tee -a $LOG
echo "===== COINTELPRO =====" | tee -a $LOG
DIR="$BASE/covert-operations/cointelpro"
mkdir -p "$DIR"
dl "https://vault.fbi.gov/cointel-pro/cointel-pro-black-extremist/cointelpro-black-extremists-part-01-of/at_download/file" \
"$DIR/cointelpro-black-extremists-part1.pdf" "COINTELPRO Black Extremists Pt1"
dl "https://vault.fbi.gov/cointel-pro/cointel-pro-white-hate/cointelpro-white-hate-groups-part-01-of/at_download/file" \
"$DIR/cointelpro-white-hate-part1.pdf" "COINTELPRO White Hate Pt1"
dl "https://vault.fbi.gov/cointel-pro/new-left/cointelpro-new-left-part-01-of/at_download/file" \
"$DIR/cointelpro-new-left-part1.pdf" "COINTELPRO New Left Pt1"
echo "" | tee -a $LOG
echo "===== STARGATE PROGRAM =====" | tee -a $LOG
DIR="$BASE/covert-operations/stargate-program"
mkdir -p "$DIR"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00788R001700210016-5.pdf" \
"$DIR/stargate-remote-viewing-session-1.pdf" "Stargate RV Session 1"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00789R003100030001-4.pdf" \
"$DIR/stargate-remote-viewing-session-2.pdf" "Stargate RV Session 2"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00787R000500250001-0.pdf" \
"$DIR/stargate-project-sun-streak.pdf" "Stargate Sun Streak"
dl "https://www.cia.gov/readingroom/docs/CIA-RDP96-00788R001900680001-4.pdf" \
"$DIR/stargate-evaluation.pdf" "Stargate Evaluation"
echo "" | tee -a $LOG
echo "===== OPERATION NORTHWOODS =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-northwoods"
mkdir -p "$DIR"
dl "https://nsarchive2.gwu.edu/news/20010430/northwoods.pdf" \
"$DIR/operation-northwoods.pdf" "Operation Northwoods"
# Backup from archive.org
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/news/20010430/northwoods.pdf" \
"$DIR/operation-northwoods.pdf" "Operation Northwoods (Archive)"
echo "" | tee -a $LOG
echo "===== OPERATION MOCKINGBIRD =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-mockingbird"
mkdir -p "$DIR"
# Church Committee Report on CIA media manipulation
dl "https://www.intelligence.senate.gov/sites/default/files/94755_I.pdf" \
"$DIR/church-committee-book1.pdf" "Church Committee Book I"
dl "https://www.intelligence.senate.gov/sites/default/files/94755_II.pdf" \
"$DIR/church-committee-book2.pdf" "Church Committee Book II"
dl "https://www.intelligence.senate.gov/sites/default/files/94755_III.pdf" \
"$DIR/church-committee-book3.pdf" "Church Committee Book III"
echo "" | tee -a $LOG
echo "===== OPERATION PAPERCLIP =====" | tee -a $LOG
DIR="$BASE/covert-operations/operation-paperclip"
mkdir -p "$DIR"
dl "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB146/doc1.pdf" \
"$DIR/paperclip-jioa-memo.pdf" "Paperclip JIOA Memo"
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/NSAEBB/NSAEBB146/doc1.pdf" \
"$DIR/paperclip-jioa-memo.pdf" "Paperclip JIOA (Archive)"
###############################################
# SECTION 3: GOVERNMENT
###############################################
echo "" | tee -a $LOG
echo "===== JFK ASSASSINATION =====" | tee -a $LOG
DIR="$BASE/government/jfk-assassination"
mkdir -p "$DIR"
# Warren Commission
dl "https://www.archives.gov/research/jfk/warren-commission-report/warren-commission-report.pdf" \
"$DIR/warren-commission-report.pdf" "Warren Commission Report"
# Try history-matters for Warren Commission
dl "https://history-matters.com/archive/jfk/wc/wr/pdf/WC_Vol0.pdf" \
"$DIR/warren-commission-vol0.pdf" "Warren Commission Vol 0"
dl "https://www.maryferrell.org/showDoc.html?docId=946" \
"$DIR/jfk-mary-ferrell.pdf" "JFK Mary Ferrell"
echo "" | tee -a $LOG
echo "===== PENTAGON PAPERS =====" | tee -a $LOG
DIR="$BASE/government/pentagon-papers"
mkdir -p "$DIR"
dl "https://www.archives.gov/research/pentagon-papers" \
"$DIR/pentagon-papers-nara.pdf" "Pentagon Papers NARA"
dl "https://media.nara.gov/research/pentagon-papers/Pentagon-Papers-Part-I.pdf" \
"$DIR/pentagon-papers-part1.pdf" "Pentagon Papers Part I"
echo "" | tee -a $LOG
echo "===== WATERGATE =====" | tee -a $LOG
DIR="$BASE/government/watergate"
mkdir -p "$DIR"
dl "https://vault.fbi.gov/watergate/watergate-part-01-of/at_download/file" \
"$DIR/watergate-fbi-part1.pdf" "Watergate FBI Part 1"
echo "" | tee -a $LOG
echo "===== IRAN-CONTRA =====" | tee -a $LOG
DIR="$BASE/government/iran-contra"
mkdir -p "$DIR"
dl "https://nsarchive2.gwu.edu/NSAEBB/NSAEBB210/iran-contra-guide.pdf" \
"$DIR/iran-contra-guide.pdf" "Iran-Contra Guide"
dl "https://web.archive.org/web/2024/https://nsarchive2.gwu.edu/NSAEBB/NSAEBB210/iran-contra-guide.pdf" \
"$DIR/iran-contra-guide.pdf" "Iran-Contra Guide (Archive)"
echo "" | tee -a $LOG
echo "===== CIA TORTURE REPORT =====" | tee -a $LOG
DIR="$BASE/government/cia-torture-report"
mkdir -p "$DIR"
# SSCI Torture Report Executive Summary (525 pages)
dl "https://www.intelligence.senate.gov/sites/default/files/publications/CRPT-113srpt288.pdf" \
"$DIR/ssci-torture-report-executive-summary.pdf" "SSCI Torture Report"
# Feinstein CIA study
dl "https://www.feinstein.senate.gov/public/_cache/files/7/c/7c85429a-ec38-4bb5-968f-289799bf6d0e/D87288C34A6D9FF736F9459ABCF83210.sscistudy1.pdf" \
"$DIR/feinstein-cia-study.pdf" "Feinstein CIA Study"
###############################################
# SUMMARY
###############################################
echo "" | tee -a $LOG
echo "========== DOWNLOAD V2 SUMMARY ==========" | tee -a $LOG
echo "" | tee -a $LOG
for dir in $(find $BASE -type d | sort); do
count=$(find "$dir" -maxdepth 1 -type f \( -name '*.pdf' -o -name '*.PDF' \) | wc -l)
if [ "$count" -gt 0 ]; then
size=$(du -sh "$dir" 2>/dev/null | cut -f1)
echo " $dir: $count files ($size)" | tee -a $LOG
fi
done
echo "" | tee -a $LOG
# Include UK existing docs in total
TOTAL_FILES=$(find $BASE -type f \( -name '*.pdf' -o -name '*.PDF' \) | wc -l)
TOTAL_SIZE=$(du -sh $BASE 2>/dev/null | cut -f1)
echo "TOTAL: $TOTAL_FILES PDF files, $TOTAL_SIZE total" | tee -a $LOG
echo "=== Download V2 complete - $(date) ===" | tee -a $LOG
Reference in a new issue View git blame Copy permalink