podcasts

Latest version of the feed fetching script:

import json
import random
import sys
import xml.etree.ElementTree as ElementTree

import requests

if __name__ == "__main__":
    # Feeds file expected JSON format:
    # [
    #    {
    #       "name": "...",
    #       "inactive": false,     # optional, if true the feed won't be fetched assuming the cached version won't change
    #       "description": "...",  # optional, if missing the original one will be kept
    #       "urls": [ 
    #           {
    #               "source": "...",
    #               "url": "..."
    #           }, ...
    #       ]
    #    }, ...
    # ]
    feeds_file = sys.argv[1]
    output_dir = sys.argv[2]

    print("\n#### Fetching feeds...")
    with open(feeds_file, newline='') as file:
        feeds = json.load(file)
        print("Loaded feeds file")

        for feed in feeds:
            # Do not fetch inactive feeds, kept as archives
            if 'inactive' in feed and feed['inactive'] is True:
                print(f"## Skipping inactive feed {feed['name']}")
            else:
                print(f"## Processing {feed['name']}...")
                sources = list(enumerate(feed['sources']))
                # Shuffle the URLs so we don't always pick the first one
                if len(sources) > 1:
                    random.shuffle(sources)

                response = None
                headers = {'User-Agent': 'AntennaPod/3.7.0'}
                # Try fetching the feed with each of the available URLs
                for i, source in sources:
                    print(f"Attempting to fetch {feed['name']} from source #{i}...")
                    url = source['url']
                    try:
                        response = requests.get(url, headers=headers)
                        if response.status_code == 200:
                            print(f"Fetched {feed['name']}")
                            break
                        else:
                            print(
                                f"ERROR: {feed['name']} URL #{i} returned error: {response.status_code} {response.content}")
                    except Exception as e:
                        print(f"ERROR: network error while fetching {feed['name']} with URL #{i}: ", e)
                if response is None or response.status_code != 200:
                    print(f"ERROR: failed to fetch {feed['name']}! No URLS worked")
                    continue

                try:
                    root = ElementTree.fromstring(response.content)
                    # Replace the description since it often contains PII
                    if 'description' in feed:
                        root[0].find('description').text = feed['description']

                    ElementTree.ElementTree(root).write(f"{output_dir}/{feed['name']}.xml")
                    print(f"Processed and saved {feed['name']}")
                except Exception as e:
                    print(f"ERROR: failed to process feed {feed['name']}:", e)