feat: use icalendar instead manual template (#9187)

* feat: use icalendar instead manual template

* avoid code duplication

* code cleanup

* ruff ruff

* remove comments

* add custom field with meeting's local Time zone

* more code cleanup

* remove unused template for ical

* pyflakes: remove unused imports and vars

* improve tests and code coverage

* remove commented line

* change URL in ical to use session material page

Author: rudimatz

ietf/meeting/tests_views.py

@@ -15,6 +15,7 @@
 from unittest.mock import call, patch, PropertyMock
 from pyquery import PyQuery
 from lxml.etree import tostring
+from icalendar import Calendar
 from io import StringIO, BytesIO
 from bs4 import BeautifulSoup
 from urllib.parse import urlparse, urlsplit
@@ -384,9 +385,6 @@ def test_meeting_agenda(self):
         r = self.client.get(ical_url)
 
         assert_ical_response_is_valid(self, r)
-        self.assertContains(r, "BEGIN:VTIMEZONE")
-        self.assertContains(r, "END:VTIMEZONE")
-        self.assertContains(r, meeting.time_zone, msg_prefix="time_zone should appear in its original case")
         self.assertNotEqual(
             meeting.time_zone,
             meeting.time_zone.lower(),
@@ -405,21 +403,32 @@ def test_meeting_agenda(self):
         assert_ical_response_is_valid(self, r)
         self.assertContains(r, session.group.acronym)
         self.assertContains(r, session.group.name)
-        self.assertContains(r, session.remote_instructions)
-        self.assertContains(r, slot.location.name)
-        self.assertContains(r, 'https://onsite.example.com')
-        self.assertContains(r, 'https://meetecho.example.com')
-        self.assertContains(r, "BEGIN:VTIMEZONE")
-        self.assertContains(r, "END:VTIMEZONE")        
 
-        self.assertContains(r, session.agenda().get_href())
-        self.assertContains(
-            r,
+        cal = Calendar.from_ical(r.content)
+        events = [component for component in cal.walk() if component.name == "VEVENT"]
+
+        self.assertEqual(len(events), 2)
+        self.assertIn(session.remote_instructions, events[0].get('description'))
+        self.assertIn("Onsite tool: https://onsite.example.com", events[0].get('description'))
+        self.assertIn("Meetecho: https://meetecho.example.com", events[0].get('description'))
+        self.assertIn(f"Agenda {session.agenda().get_href()}", events[0].get('description'))
+        session_materials_url = settings.IDTRACKER_BASE_URL + urlreverse(
+            'ietf.meeting.views.session_details',
+            kwargs=dict(num=meeting.number, acronym=session.group.acronym)
+        )
+        self.assertIn(f"Session materials: {session_materials_url}", events[0].get('description'))
+        self.assertIn(
             urlreverse(
                 'ietf.meeting.views.session_details',
                 kwargs=dict(num=meeting.number, acronym=session.group.acronym)),
-            msg_prefix='ical should contain link to meeting materials page for session')
+            events[0].get('description'))
+        self.assertEqual(
+            session_materials_url,
+            events[0].get('url')
+        )
 
+        self.assertContains(r, f"LOCATION:{slot.location.name}")
+        
         # Floor Plan
         r = self.client.get(urlreverse('floor-plan', kwargs=dict(num=meeting.number)))
         self.assertEqual(r.status_code, 200)
@@ -1049,32 +1058,36 @@ def test_group_ical(self):
         s1 = Session.objects.filter(meeting=meeting, group__acronym="mars").first()
         a1 = s1.official_timeslotassignment()
         t1 = a1.timeslot
+
         # Create an extra session
         t2 = TimeSlotFactory.create(
             meeting=meeting,
-            time=meeting.tz().localize(
+            time=pytz.utc.localize(
                 datetime.datetime.combine(meeting.date, datetime.time(11, 30))
             )
         )
+     
         s2 = SessionFactory.create(meeting=meeting, group=s1.group, add_to_schedule=False)
         SchedTimeSessAssignment.objects.create(timeslot=t2, session=s2, schedule=meeting.schedule)
-        #
+        
         url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, })
         r = self.client.get(url)
         assert_ical_response_is_valid(self,
                                       r,
                                       expected_event_summaries=['mars - Martian Special Interest Group'],
                                       expected_event_count=2)
-        self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        self.assertContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        #
+        self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTEND:{(t1.time + t1.duration).strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertContains(r, f"DTEND:{(t2.time + t2.duration).strftime('%Y%m%dT%H%M%SZ')}")
+
         url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'session_id':s1.id, })
         r = self.client.get(url)
         assert_ical_response_is_valid(self, r,
                                       expected_event_summaries=['mars - Martian Special Interest Group'],
                                       expected_event_count=1)
-        self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S'))
-        self.assertNotContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S'))
+        self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}")
+        self.assertNotContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}")
 
     def test_parse_agenda_filter_params(self):
         def _r(show=(), hide=(), showtypes=(), hidetypes=()):

ietf/meeting/views.py

@@ -118,6 +118,9 @@
     UploadAgendaForm, UploadBlueSheetForm, UploadMinutesForm, UploadSlidesForm,
     UploadNarrativeMinutesForm)
 
+from icalendar import Calendar, Event
+from ietf.doc.templatetags.ietf_filters import absurl
+
 request_summary_exclude_group_types = ['team']
 
 
@@ -137,6 +140,10 @@ def send_interim_change_notice(request, meeting):
     message.related_groups.add(group)
     send_mail_message(request, message)
 
+def parse_ical_line_endings(ical):
+    """Parse icalendar line endings to ensure they are RFC 5545 compliant"""
+    return re.sub(r'\r(?!\n)|(?<!\r)\n', '\r\n', ical)
+
 # -------------------------------------------------
 # View Functions
 # -------------------------------------------------
@@ -1982,8 +1989,10 @@ def agenda_by_type_ics(request,num=None,type=None):
     ).order_by('session__type__slug','timeslot__time')
     if type:
         assignments = assignments.filter(session__type__slug=type)
-    updated = meeting.updated()
-    return render(request,"meeting/agenda.ics",{"schedule":schedule,"updated":updated,"assignments":assignments},content_type="text/calendar")
+
+    return render_icalendar(schedule, assignments)
+
+
 
 def session_draft_list(num, acronym):
     try:
@@ -2103,6 +2112,125 @@ def ical_session_status(assignment):
     else:
         return "CONFIRMED"
 
+def render_icalendar(schedule, assignments):
+    ical_content = generate_agenda_ical(schedule, assignments)
+    return HttpResponse(ical_content, content_type="text/calendar")
+
+def generate_agenda_ical(schedule, assignments):
+    """Generate iCalendar using the icalendar library"""
+
+    cal = Calendar()
+    cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN")
+    cal.add("version", "2.0")
+    cal.add("method", "PUBLISH")
+
+    for item in assignments:
+        event = Event()
+
+        uid = f"ietf-{schedule.meeting.number}-{item.timeslot.pk}-{item.session.group.acronym}"
+        event.add("uid", uid)
+
+        # add custom field with meeting's local TZ
+        event.add("x-meeting-tz", schedule.meeting.time_zone)
+
+        if item.session.name:
+            summary = item.session.name
+        else:
+            group = item.session.group_at_the_time()
+            summary = f"{group.acronym} - {group.name}"
+
+        if item.session.agenda_note:
+            summary += f" ({item.session.agenda_note})"
+
+        event.add("summary", summary)
+
+        if item.timeslot.show_location and item.timeslot.get_location():
+            event.add("location", item.timeslot.get_location())
+
+        if item.session and hasattr(item.session, "current_status"):
+            status = ical_session_status(item)
+        else:
+            status = ""
+        event.add("status", status)
+
+        event.add("class", "PUBLIC")
+
+        event.add("dtstart", item.timeslot.utc_start_time())
+        event.add("dtend", item.timeslot.utc_end_time())
+
+        # DTSTAMP: when the event was created or last modified (in UTC)
+        dtstamp = item.timeslot.modified.astimezone(pytz.UTC)
+        event.add("dtstamp", dtstamp)
+
+        description_parts = [item.timeslot.name]
+
+        if item.session.agenda_note:
+            description_parts.append(f"Note: {item.session.agenda_note}")
+
+        if hasattr(item.session, "onsite_tool_url") and callable(
+            item.session.onsite_tool_url
+        ):
+            onsite_url = item.session.onsite_tool_url()
+            if onsite_url:
+                description_parts.append(f"Onsite tool: {onsite_url}")
+
+        if hasattr(item.session, "video_stream_url") and callable(
+            item.session.video_stream_url
+        ):
+            video_url = item.session.video_stream_url()
+            if video_url:
+                description_parts.append(f"Meetecho: {video_url}")
+
+        if (
+            item.timeslot.location
+            and hasattr(item.timeslot.location, "webex_url")
+            and callable(item.timeslot.location.webex_url)
+            and item.timeslot.location.webex_url() is not None
+        ):
+            description_parts.append(f"Webex: {item.timeslot.location.webex_url()}")
+
+        if item.session.remote_instructions:
+            description_parts.append(
+                f"Remote instructions: {item.session.remote_instructions}"
+            )
+
+        try:
+            materials_url = absurl(
+                "ietf.meeting.views.session_details",
+                num=schedule.meeting.number,
+                acronym=item.session.group.acronym,
+            )
+            description_parts.append(f"Session materials: {materials_url}")
+            event.add("url", materials_url)
+        except:
+            pass
+
+        if (
+            hasattr(schedule.meeting, "get_number")
+            and schedule.meeting.get_number() is not None
+        ):
+            try:
+                agenda_url = absurl("agenda", num=schedule.meeting.number)
+                description_parts.append(
+                    f"See in schedule: {agenda_url}#row-{item.slug()}"
+                )
+            except:
+                pass
+
+        agenda = item.session.agenda()
+        if agenda and hasattr(agenda, "get_versionless_href"):
+            agenda_url = agenda.get_versionless_href()
+            description_parts.append(f"{agenda.type} {agenda_url}")
+
+        # Join all description parts with 2 newlines
+        description = "\n\n".join(description_parts)
+        event.add("description", description)
+
+        # Add event to calendar
+        cal.add_component(event)
+
+    return cal.to_ical().decode("utf-8")
+
 def parse_agenda_filter_params(querydict):
     """Parse agenda filter parameters from a request"""
     if len(querydict) == 0:
@@ -2154,7 +2282,6 @@ def agenda_ical(request, num=None, acronym=None, session_id=None):
     else:
         meeting = get_meeting(num, type_in=None)  # get requested meeting, whatever its type
     schedule = get_schedule(meeting)
-    updated = meeting.updated()
 
     if schedule is None and acronym is None and session_id is None:
         raise Http404
@@ -2180,15 +2307,7 @@ def agenda_ical(request, num=None, acronym=None, session_id=None):
     elif session_id:
         assignments = [ a for a in assignments if a.session_id == int(session_id) ]
 
-    for a in assignments:
-        if a.session:
-            a.session.ical_status = ical_session_status(a)
-
-    return render(request, "meeting/agenda.ics", {
-        "schedule": schedule,
-        "assignments": assignments,
-        "updated": updated
-    }, content_type="text/calendar")
+    return render_icalendar(schedule, assignments)
 
 @cache_page(15 * 60)
 def agenda_json(request, num=None):
@@ -4138,7 +4257,7 @@ def upcoming_ical(request):
         'assignments': assignments,
         'ietfs': ietfs,
     }, request=request)
-    response = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", response)
+    response = parse_ical_line_endings(response)
 
     response = HttpResponse(response, content_type='text/calendar')
     response['Content-Disposition'] = 'attachment; filename="upcoming.ics"'
@@ -4764,7 +4883,7 @@ def important_dates(request, num=None, output_format=None):
             'meetings': meetings,
         }, request=request)
         # icalendar response file should have '\r\n' line endings per RFC5545
-        response = HttpResponse(re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", ics), content_type='text/calendar')
+        response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar')
         response['Content-Disposition'] = 'attachment; filename="important-dates.ics"'
         return response
 
@@ -5198,3 +5317,4 @@ def import_session_minutes(request, session_id, num):
             'contents_unchanged': not contents_changed,
         },
     )
+

ietf/templates/meeting/agenda.ics

@@ -42,6 +42,7 @@ gunicorn>=20.1.0
 hashids>=1.3.1
 html2text>=2020.1.16    # Used only to clean comment field of secr/sreq
 html5lib>=1.1    # Only used in tests
+icalendar>=5.0.0 
 inflect>= 6.0.2
 jsonfield>=3.1.0,<3.2.0  # 3.2.0 needs py3.10; deprecated-replace with Django JSONField
 jsonschema[format]>=4.2.1