diff --git a/cleanup.sh b/cleanup.sh new file mode 100755 index 0000000..61fadbf --- /dev/null +++ b/cleanup.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Cleanup unnecessary files from the project + +echo "๐Ÿงน Cleaning up unnecessary files..." + +# Remove Python cache files +echo "Removing Python cache files..." +find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null +find . -type f -name "*.pyc" -delete 2>/dev/null +find . -type f -name "*.pyo" -delete 2>/dev/null + +# Remove local virtual environments (should use Docker) +echo "Removing local virtual environments..." +rm -rf backend/env 2>/dev/null +rm -rf news_crawler/env 2>/dev/null +rm -rf news_sender/env 2>/dev/null +rm -rf transport_crawler/env 2>/dev/null + +# Remove local database files (should use MongoDB in Docker) +echo "Removing local database files..." +rm -f backend/munich_news.db 2>/dev/null +rm -f backend/*.db 2>/dev/null + +# Remove macOS files +echo "Removing macOS files..." +find . -name ".DS_Store" -delete 2>/dev/null + +# Remove editor files +echo "Removing editor backup files..." +find . -name "*~" -delete 2>/dev/null +find . -name "*.swp" -delete 2>/dev/null +find . -name "*.swo" -delete 2>/dev/null + +echo "โœ… Cleanup complete!" diff --git a/docker-compose.yml b/docker-compose.yml index 437b700..99393d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,6 +206,10 @@ services: restart: unless-stopped ports: - "3000:3000" + # Traefik Configuration (commented out - uncomment when using Traefik) + # Remove port exposure when using Traefik + # ports: + # - "3000:3000" environment: - API_URL=http://backend:5001 - PORT=3000 diff --git a/fix-categories.js b/fix-categories.js deleted file mode 100644 index 58ac0b9..0000000 --- a/fix-categories.js +++ /dev/null @@ -1,56 +0,0 @@ -// MongoDB script to fix article categories -// Run with: docker exec -i munich-news-mongodb mongosh -u admin -p changeme --authenticationDatabase admin < fix-categories.js - -use munich_news - -print("=== RSS Feeds and their categories ==="); -db.rss_feeds.find({}, {name: 1, category: 1, _id: 0}).forEach(feed => { - print(`${feed.name}: ${feed.category || 'NO CATEGORY'}`); -}); - -print("\n=== Current article category distribution ==="); -db.articles.aggregate([ - {$group: {_id: "$category", count: {$sum: 1}}}, - {$sort: {count: -1}} -]).forEach(result => { - print(`${result._id || 'null'}: ${result.count} articles`); -}); - -print("\n=== Fixing null categories ==="); - -// Update articles based on their RSS feed source -var feedsUpdated = 0; -db.rss_feeds.find().forEach(function(feed) { - if (feed.category) { - var result = db.articles.updateMany( - {source: feed.name, category: null}, - {$set: {category: feed.category}} - ); - if (result.modifiedCount > 0) { - print(`Updated ${result.modifiedCount} articles from ${feed.name} to category: ${feed.category}`); - feedsUpdated += result.modifiedCount; - } - } -}); - -// Set remaining null categories to 'general' -var remainingNull = db.articles.updateMany( - {category: null}, - {$set: {category: "general"}} -); - -if (remainingNull.modifiedCount > 0) { - print(`Set ${remainingNull.modifiedCount} remaining null articles to 'general'`); -} - -print(`\nTotal articles updated: ${feedsUpdated + remainingNull.modifiedCount}`); - -print("\n=== Updated article category distribution ==="); -db.articles.aggregate([ - {$group: {_id: "$category", count: {$sum: 1}}}, - {$sort: {count: -1}} -]).forEach(result => { - print(`${result._id || 'null'}: ${result.count} articles`); -}); - -print("\nโœ“ Done!"); diff --git a/news_sender/newsletter_template.html b/news_sender/newsletter_template.html index bdf11dc..82807d8 100644 --- a/news_sender/newsletter_template.html +++ b/news_sender/newsletter_template.html @@ -83,6 +83,101 @@ + + + +

+ ๐Ÿ“‹ TL;DR - Quick Summary +

+

+ Here's everything in one sentence each: +

+ + {% for section in category_sections %} + {% for article in section.articles %} +
+ {{ loop.index }}. +

+ {{ article.title_en if article.title_en else article.title }} โ€” + {{ article.summary.split('.')[0] }}. +

+
+ {% endfor %} + {% endfor %} + + + + {% if transport_disruptions and transport_disruptions|length > 0 %} + + + +
+ + + + + + +

+ ๐Ÿš† S-Bahn Disruptions Today +

+

+ Current service disruptions affecting Munich S-Bahn: +

+ + {% for disruption in transport_disruptions %} + + + + + +
+ +

+ {{ disruption.severity_icon }} {{ disruption.lines_str }} +

+ + +

+ {{ disruption.title }} +

+ + + {% if disruption.description %} +

+ {{ disruption.description }} +

+ {% endif %} + + + {% if disruption.start_time_str or disruption.end_time_str %} +

+ โฐ + {% if disruption.start_time_str %} + From {{ disruption.start_time_str }} + {% endif %} + {% if disruption.end_time_str %} + until {{ disruption.end_time_str }} + {% endif %} +

+ {% endif %} +
+ {% endfor %} + +

+ ๐Ÿ’ก Plan your commute accordingly. Check MVG.de for real-time updates. +

+ + + {% endif %} + + + + +
+ + + {% for section in category_sections %} @@ -195,39 +290,6 @@ - - - -

- ๐Ÿ“‹ TL;DR - Quick Summary -

-

- Here's everything in one sentence each: -

- - {% for section in category_sections %} - {% for article in section.articles %} -
- {{ loop.index }}. -

- {{ article.title_en if article.title_en else article.title }} โ€” - {{ article.summary.split('.')[0] }}. -

-
- {% endfor %} - {% endfor %} - - - - - - -
- - - - - @@ -245,72 +307,6 @@
- - - {% if transport_disruptions and transport_disruptions|length > 0 %} - - - -
- - - - - - -

- ๐Ÿš† S-Bahn Disruptions Today -

-

- Current service disruptions affecting Munich S-Bahn: -

- - {% for disruption in transport_disruptions %} - - - - - -
- -

- {{ disruption.severity_icon }} {{ disruption.lines_str }} -

- - -

- {{ disruption.title }} -

- - - {% if disruption.description %} -

- {{ disruption.description }} -

- {% endif %} - - - {% if disruption.start_time_str or disruption.end_time_str %} -

- โฐ - {% if disruption.start_time_str %} - From {{ disruption.start_time_str }} - {% endif %} - {% if disruption.end_time_str %} - until {{ disruption.end_time_str }} - {% endif %} -

- {% endif %} -
- {% endfor %} - -

- ๐Ÿ’ก Plan your commute accordingly. Check MVG.de for real-time updates. -

- - - {% endif %} - diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..5a964b8 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,32 @@ +# Utility Scripts + +This folder contains utility scripts for testing and managing the Munich News Daily system. + +## Available Scripts + +### Ollama / GPU +- `setup-ollama-model.sh` - Pull and setup the Ollama model (used by Docker) +- `configure-ollama.sh` - Configure Ollama settings +- `pull-ollama-model.sh` - Manually pull Ollama model +- `start-with-gpu.sh` - Start services with GPU support +- `check-gpu.sh` - Check GPU availability +- `check-gpu-api.sh` - Check GPU via API +- `diagnose-gpu.sh` - Diagnose GPU issues +- `test-ollama-setup.sh` - Test Ollama configuration + +### Testing +- `test-mongodb-connectivity.sh` - Test MongoDB connection +- `test-newsletter-api.sh` - Test newsletter API endpoints +- `check-articles.sh` - Check articles in database + +## Usage + +Make scripts executable: +```bash +chmod +x scripts/*.sh +``` + +Run a script: +```bash +./scripts/script-name.sh +``` diff --git a/check-articles.sh b/scripts/check-articles.sh similarity index 100% rename from check-articles.sh rename to scripts/check-articles.sh diff --git a/check-gpu-api.sh b/scripts/check-gpu-api.sh similarity index 100% rename from check-gpu-api.sh rename to scripts/check-gpu-api.sh diff --git a/check-gpu.sh b/scripts/check-gpu.sh similarity index 100% rename from check-gpu.sh rename to scripts/check-gpu.sh diff --git a/configure-ollama.sh b/scripts/configure-ollama.sh similarity index 100% rename from configure-ollama.sh rename to scripts/configure-ollama.sh diff --git a/diagnose-gpu.sh b/scripts/diagnose-gpu.sh similarity index 100% rename from diagnose-gpu.sh rename to scripts/diagnose-gpu.sh diff --git a/pull-ollama-model.sh b/scripts/pull-ollama-model.sh similarity index 100% rename from pull-ollama-model.sh rename to scripts/pull-ollama-model.sh diff --git a/start-with-gpu.sh b/scripts/start-with-gpu.sh similarity index 100% rename from start-with-gpu.sh rename to scripts/start-with-gpu.sh diff --git a/test-mongodb-connectivity.sh b/scripts/test-mongodb-connectivity.sh similarity index 100% rename from test-mongodb-connectivity.sh rename to scripts/test-mongodb-connectivity.sh diff --git a/test-newsletter-api.sh b/scripts/test-newsletter-api.sh similarity index 100% rename from test-newsletter-api.sh rename to scripts/test-newsletter-api.sh diff --git a/test-ollama-setup.sh b/scripts/test-ollama-setup.sh similarity index 100% rename from test-ollama-setup.sh rename to scripts/test-ollama-setup.sh