diff --git a/README.md b/README.md new file mode 100644 index 0000000..c883e7c --- /dev/null +++ b/README.md @@ -0,0 +1,62 @@ +# MeetupScraper + +A lightweight Python library for scraping upcoming events from [meetup.com](https://meetup.com) groups. + +## Features + +- Fetch upcoming events for any public Meetup group +- Filter events by name using regular expressions +- Structured event data with title, date, URL, and venue information +- Venue details fetched via Meetup's GraphQL API +- Search events across all groups in a specific city +- Automatic space-to-hyphen conversion for meetup group names +- Handles both datetime and date-only event start times + +## Installation + +```bash +pip install -e . +``` + +Or install dependencies manually: + +```bash +pip install requests lxml python-dateutil icalendar gql aiohttp +``` + +## Running the Script + +Run the [`search_events.py`](search_events.py) script directly with the desired arguments: + +```bash +# Search by meetup group +python search_events.py --meetup Hakierspejs-Łodź --regex Radio + +# Search by city +python search_events.py --city Warsaw + +# Search by city with regex filter and more groups +python search_events.py --city "Krakow" --regex Python --max-groups 20 +``` + +### CLI Arguments + +| Flag | Description | Required | +|------|-------------|----------| +| `--meetup`, `-m` | Meetup group name or URL slug (e.g., `Hakierspejs-Łodź`) | One of `--meetup` or `--city` | +| `--city`, `-c` | City name to search for events (e.g., `Warsaw`, `Krakow`) | One of `--meetup` or `--city` | +| `--regex`, `-r` | Filter events by title using a regular expression | No | +| `--max-groups` | Maximum number of groups to search when using `--city` (default: 10) | No | +| `--debug` | Enable debug logging | No | + +## Dependencies + +- `requests` — HTTP requests for iCal feed +- `icalendar` — iCal parsing +- `gql` + `aiohttp` — GraphQL client for venue data +- `lxml` — XML/HTML parsing (transitive dependency) +- `python-dateutil` — Date parsing (transitive dependency) + +## License + +WTFPL diff --git a/gql_data.py b/gql_data.py index 466287c..5780203 100644 --- a/gql_data.py +++ b/gql_data.py @@ -1,8 +1,11 @@ from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport import json +import requests + def get_gql_data(event_id): + """Fetch event details from Meetup's GraphQL API.""" transport = AIOHTTPTransport(url="https://www.meetup.com/gql2") client = Client(transport=transport) @@ -29,8 +32,62 @@ def get_gql_data(event_id): } """) - result = client.execute(query, variable_values={"eventId": str(event_id)}) - return(result) + return result + +def geocode_city(city_name): + """Convert city name to latitude/longitude using Nominatim.""" + url = "https://nominatim.openstreetmap.org/search" + params = { + "q": city_name, + "format": "json", + "limit": 1, + } + headers = { + "User-Agent": "meetupscraper/1.0", + } + response = requests.get(url, params=params, headers=headers) + data = response.json() + + if not data: + raise ValueError(f"Could not geocode city: {city_name}") + + return float(data[0]["lat"]), float(data[0]["lon"]) + + +def get_groups_by_city(city_name, limit=20): + """Search for meetup groups in a given city using Meetup's GraphQL API.""" + lat, lon = geocode_city(city_name) + + transport = AIOHTTPTransport(url="https://www.meetup.com/gql2") + + client = Client(transport=transport) + + query = gql(""" + query SearchGroups($query: String!, $first: Int!, $lat: Float!, $lon: Float!) { + groupSearch(first: $first, filter: {query: $query, lat: $lat, lon: $lon}) { + edges { + node { + id + name + urlname + description + city + country + } + } + totalCount + } + } + """) + + result = client.execute(query, variable_values={ + "query": city_name, + "first": limit, + "lat": lat, + "lon": lon, + }) + + return result diff --git a/meetupscraper.egg-info/PKG-INFO b/meetupscraper.egg-info/PKG-INFO new file mode 100644 index 0000000..efe2f32 --- /dev/null +++ b/meetupscraper.egg-info/PKG-INFO @@ -0,0 +1,30 @@ +Metadata-Version: 2.4 +Name: meetupscraper +Version: 0.11.2 +Summary: A web scraper for meetup.com +Home-page: http://github.com/hakierspejs/meetupscraper +Author: Hakiespejs Łódź +Author-email: d33tah@gmail.com +License: WTFPL +Requires-Python: >=3.7 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: requests +Requires-Dist: lxml +Requires-Dist: python-dateutil +Requires-Dist: icalendar +Requires-Dist: gql +Requires-Dist: aiohttp +Requires-Dist: json +Dynamic: author +Dynamic: author-email +Dynamic: description +Dynamic: description-content-type +Dynamic: home-page +Dynamic: license +Dynamic: license-file +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary + +Allows you to access events tied to a specific meetup group in a structured way diff --git a/meetupscraper.egg-info/SOURCES.txt b/meetupscraper.egg-info/SOURCES.txt new file mode 100644 index 0000000..182803d --- /dev/null +++ b/meetupscraper.egg-info/SOURCES.txt @@ -0,0 +1,10 @@ +LICENSE +README.md +setup.py +meetupscraper/__init__.py +meetupscraper.egg-info/PKG-INFO +meetupscraper.egg-info/SOURCES.txt +meetupscraper.egg-info/dependency_links.txt +meetupscraper.egg-info/entry_points.txt +meetupscraper.egg-info/requires.txt +meetupscraper.egg-info/top_level.txt \ No newline at end of file diff --git a/meetupscraper.egg-info/dependency_links.txt b/meetupscraper.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/meetupscraper.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/meetupscraper.egg-info/entry_points.txt b/meetupscraper.egg-info/entry_points.txt new file mode 100644 index 0000000..8c90b51 --- /dev/null +++ b/meetupscraper.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +meetupscraper = search_events:main diff --git a/meetupscraper.egg-info/requires.txt b/meetupscraper.egg-info/requires.txt new file mode 100644 index 0000000..127eb00 --- /dev/null +++ b/meetupscraper.egg-info/requires.txt @@ -0,0 +1,7 @@ +requests +lxml +python-dateutil +icalendar +gql +aiohttp +json diff --git a/meetupscraper.egg-info/top_level.txt b/meetupscraper.egg-info/top_level.txt new file mode 100644 index 0000000..5d8167e --- /dev/null +++ b/meetupscraper.egg-info/top_level.txt @@ -0,0 +1 @@ +meetupscraper diff --git a/meetupscraper/__init__.py b/meetupscraper/__init__.py index 7718fa3..e64a944 100644 --- a/meetupscraper/__init__.py +++ b/meetupscraper/__init__.py @@ -6,7 +6,7 @@ from datetime import timezone import icalendar import requests -from gql_data import get_gql_data +from gql_data import get_gql_data, get_groups_by_city import json @@ -62,23 +62,9 @@ def get_upcoming_events(meetup_name, name_regex=None): LOGGER.info("Skipping event %r", event_name) continue - #location = str(component.get('location', '')) - #venue_name = location - #venue_street = '' - # - #if ',' in location: - # venue_parts = location.split(',', 1) - # venue_name = venue_parts[0].strip() - # venue_street = venue_parts[1].strip() - # - #venue = Venue(name=venue_name, street=venue_street) - ## set to None if no address nor name - #if not venue_name and not venue_street: - # venue = None try: event_id = re.search(r".+\/(\d+)\/", event_url).groups()[0] gql_data = get_gql_data(event_id) - print(gql_data['event']['venue']) venue = Venue( name=gql_data['event']['venue']['name'], street=gql_data['event']['venue']['address'] + ', ' + gql_data['event']['venue']['city'] @@ -92,3 +78,50 @@ def get_upcoming_events(meetup_name, name_regex=None): url=event_url, venue=venue, ) + + +def get_events_by_city(city_name, name_regex=None, max_groups=10): + """ + Search for upcoming events in a given city by finding groups and aggregating their events. + + Args: + city_name: Name of the city to search for events (e.g., 'Warsaw', 'Krakow') + name_regex: Optional regex pattern to filter events by title + max_groups: Maximum number of groups to search (default 10) + + Yields: + Event objects for all upcoming events in the city + """ + LOGGER.info("Searching for groups in city: %s", city_name) + + # Search for groups in the city + groups_result = get_groups_by_city(city_name, limit=max_groups) + + groups = groups_result.get('groupSearch', {}).get('edges', []) + if not groups: + LOGGER.warning("No groups found for city: %s", city_name) + return + + LOGGER.info("Found %d groups in %s", len(groups), city_name) + + # Collect events from all groups + seen_events = set() # To avoid duplicates + + for group_edge in groups: + group = group_edge.get('node', {}) + urlname = group.get('urlname') + + if not urlname: + continue + + LOGGER.info("Fetching events from group: %s", group.get('name', urlname)) + + try: + for event in get_upcoming_events(urlname, name_regex=name_regex): + # Use URL as unique identifier to avoid duplicates + if event.url not in seen_events: + seen_events.add(event.url) + yield event + except Exception as e: + LOGGER.warning("Failed to fetch events from group %s: %s", urlname, e) + continue diff --git a/search_events.py b/search_events.py new file mode 100644 index 0000000..8e4faca --- /dev/null +++ b/search_events.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Command-line script to search for upcoming events on Meetup.com. + +Usage: + python search_events.py --meetup Hakierspejs-Łodź + python search_events.py --meetup Hakierspejs-Łodź --regex Radio + python search_events.py --meetup hakierspejs-lodz + python search_events.py --city Warsaw + python search_events.py --city "Krakow" --regex Python +""" + +import argparse +import logging +import sys + +import meetupscraper + + +def search_events(meetup_name: str, name_regex: str = None): + """Search for upcoming events and return results.""" + print(f"Searching events for meetup group: {meetup_name}") + if name_regex: + print(f"Filtering by regex: {name_regex}") + print("-" * 60) + + events = list(meetupscraper.get_upcoming_events(meetup_name, name_regex=name_regex)) + return events + + +def search_events_by_city(city_name: str, name_regex: str = None, max_groups: int = 10): + """Search for upcoming events in a city and return results.""" + print(f"Searching events in city: {city_name}") + if name_regex: + print(f"Filtering by regex: {name_regex}") + print(f"Max groups to search: {max_groups}") + print("-" * 60) + + events = list(meetupscraper.get_events_by_city(city_name, name_regex=name_regex, max_groups=max_groups)) + return events + + +def print_events(events): + """Print events in a readable format.""" + if not events: + print("No events found.") + return + + print(f"Found {len(events)} events:\n") + for i, event in enumerate(events, 1): + print(f"{i}. {event.title}") + print(f" URL: {event.url}") + print(f" Date: {event.date}") + if event.venue: + print(f" Venue: {event.venue.name}") + print(f" Address: {event.venue.street}") + print() + + +def main(): + parser = argparse.ArgumentParser( + description="Search for upcoming events on Meetup.com by group name or city." + ) + + # Create mutually exclusive group for meetup vs city + search_group = parser.add_mutually_exclusive_group(required=True) + search_group.add_argument( + "--meetup", "-m", + help="Meetup group name or URL slug (e.g., 'Hakierspejs-Łodź' or 'hakierspejs-lodz')", + ) + search_group.add_argument( + "--city", "-c", + help="City name to search for events (e.g., 'Warsaw', 'Krakow')", + ) + + parser.add_argument( + "--regex", "-r", + help="Filter events by title using a regular expression", + ) + parser.add_argument( + "--max-groups", + type=int, + default=10, + help="Maximum number of groups to search when using --city (default: 10)", + ) + parser.add_argument( + "--debug", + action="store_true", + help="Enable debug logging", + ) + + args = parser.parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + try: + if args.city: + events = search_events_by_city( + city_name=args.city, + name_regex=args.regex, + max_groups=args.max_groups, + ) + else: + events = search_events( + meetup_name=args.meetup, + name_regex=args.regex, + ) + print_events(events) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 36c5590..d0358c6 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,13 @@ version='0.11.2', description='A web scraper for meetup.com', long_description='Allows you to access events tied to a specific meetup group in a structured way', + long_description_content_type='text/markdown', url='http://github.com/hakierspejs/meetupscraper', author='Hakiespejs Łódź', author_email='d33tah@gmail.com', license='WTFPL', packages=['meetupscraper'], + python_requires='>=3.7', install_requires=[ 'requests', 'lxml', @@ -23,5 +25,5 @@ 'gql', 'aiohttp', 'json' - ] + ], ) diff --git a/test_gql.py b/test_gql.py new file mode 100644 index 0000000..0092011 --- /dev/null +++ b/test_gql.py @@ -0,0 +1,73 @@ +import requests +import json +import sys + +url = 'https://www.meetup.com/gql2' + +# Try different approaches +queries = [ + ("groupSearch with query only", """ + query { + groupSearch(first: 10, filter: {query: "Warsaw", lat: 52.2297, lon: 21.0122}) { + edges { + node { + name + urlname + city + country + } + } + totalCount + } + } + """), + ("groupSearch with country", """ + query { + groupSearch(first: 10, filter: {country: "PL", query: "", lat: 52.2297, lon: 21.0122}) { + edges { + node { + name + urlname + city + country + } + } + totalCount + } + } + """), + ("groupSearch with radius", """ + query { + groupSearch(first: 10, filter: {city: "Warsaw", query: "", lat: 52.2297, lon: 21.0122, radius: 50}) { + edges { + node { + name + urlname + city + country + } + } + totalCount + } + } + """), + ("groups all", """ + query { + groups(where: {first: 10}) { + name + urlname + city + country + } + } + """), +] + +with open('test_results.txt', 'w', encoding='utf-8') as f: + for name, query in queries: + response = requests.post(url, json={'query': query}) + f.write(f"=== {name} ===\n") + f.write(f"Status: {response.status_code}\n") + f.write(f"Body: {response.text[:1200]}\n\n") + +print("done") diff --git a/test_results.txt b/test_results.txt new file mode 100644 index 0000000..99e057a --- /dev/null +++ b/test_results.txt @@ -0,0 +1,16 @@ +=== groupSearch with query only === +Status: 200 +Body: {"data":{"groupSearch":{"edges":[{"node":{"name":"Impact Warsaw Family","urlname":"impact-warsaw-family","city":"Warsaw","country":"pl"}},{"node":{"name":"Warsaw Locals Community","urlname":"warsaw-locals-community","city":"Warsaw","country":"pl"}},{"node":{"name":"Warsaw Apache Airflow Meetup Group","urlname":"warsaw-apache-airflow-meetup-group","city":"Warsaw","country":"pl"}},{"node":{"name":"Warsaw Pub Crawl","urlname":"warsawpubcrawl","city":"Warsaw","country":"pl"}},{"node":{"name":"AI builders Warsaw","urlname":"ai-builders-warsaw","city":"Warsaw","country":"pl"}},{"node":{"name":"Meet & Speak - Warsaw Social & Language Exchange","urlname":"meet-speak-warsaw-social-language-exchange","city":"Warsaw","country":"pl"}},{"node":{"name":"Rust Warsaw","urlname":"rust-warsaw","city":"Warsaw","country":"pl"}},{"node":{"name":"Warsaw English speaking Club","urlname":"warsaw-english-speaking-club","city":"Warsaw","country":"pl"}},{"node":{"name":"Sweet Mokotow Neighbors","urlname":"sweet-mokotow-neighbors","city":"Warsaw","country":"pl"}},{"node":{"name":"Czyste Pędzle Warsaw","urlname":"czyste-pędzle-warsaw","city":"Warsaw","country":"pl"}}],"totalCount":10}}} + +=== groupSearch with country === +Status: 200 +Body: {"data":{"groupSearch":{"edges":[],"totalCount":0}}} + +=== groupSearch with radius === +Status: 200 +Body: {"data":{"groupSearch":{"edges":[],"totalCount":0}}} + +=== groups all === +Status: 200 +Body: {"data":{"groups":[{"name":"The Yerushalayim Amiga Meetup Group","urlname":"_fns_amiga@1","city":"Jerusalem","country":"il"},{"name":"The Washington Anime Meetup Group","urlname":"_fns_anime@57","city":"Washington","country":"us"},{"name":"Morris County Bass Player Meetup Group","urlname":"_fns_bass@16","city":"Morris Plains","country":"us"},{"name":"The Portland Birds Meetup Group","urlname":"_fns_birds@39","city":"Portland","country":"us"},{"name":"The Austin Weblogger Meetup Group","urlname":"_fns_blog@114","city":"Austin","country":"us"},{"name":"The Greater Boston Area Weblogger Meetup Group","urlname":"bloggers","city":"Cambridge","country":"us"},{"name":"The Denver Weblogger Meetup Group","urlname":"_fns_blog@142","city":"Denver","country":"us"},{"name":"CLUTCH-22","urlname":"_fns_blog@22","city":"West Lafayette","country":"us"},{"name":"The Milwaukee Weblogger Meetup Group","urlname":"_fns_blog@127","city":"Milwaukee","country":"us"},{"name":"The Minneapolis Weblogger Meetup Group","urlname":"_fns_blog@128","city":"Minneapolis","country":"us"}]}} +