From 595f59aa95b06460574a363b9bbfeb979d1b0d73 Mon Sep 17 00:00:00 2001
From: Aaron Gutierrez <aaron@frat.tech>
Date: Sat, 3 Apr 2021 00:21:02 -0700
Subject: [PATCH] Fix pagination

  - Paginate through S3 keys
  - Off-by-one in bench page navigation
---
 bench_view_template.html |  4 ++--
 pub.py                   | 27 ++++++++++++---------------
 2 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/bench_view_template.html b/bench_view_template.html
index f5f982b..25777fc 100644
--- a/bench_view_template.html
+++ b/bench_view_template.html
@@ -20,9 +20,9 @@
         {0}
       </div>
       <div class="nav">
-        <a href="{3}">Next</a>
+        <a href="{4}">Next</a>
         <a href="/">Home</a>
-        <a href="{2}">Prev</a>
+        <a href="{3}">Prev</a>
       </nav>
     </main>
     <hr>
diff --git a/pub.py b/pub.py
index 3017ef7..c99f97a 100755
--- a/pub.py
+++ b/pub.py
@@ -30,22 +30,23 @@ TYPE_MAP = {
 def current_keys():
     print('Fetching existing keys in {}'.format(BUCKET))
     existing = s3.list_objects_v2(Bucket=BUCKET)
+    keys = set([content['Key'] for content in existing['Contents']])
+    while existing['IsTruncated']:
+        existing = s3.list_objects_v2(Bucket=BUCKET, ContinuationToken=existing['NextContinuationToken'])
+        keys = keys.union(set([content['Key'] for content in existing['Contents']]))
 
-    return [content['Key'] for content in existing['Contents']]
+    return keys
 
+EXISTING_KEYS = current_keys()
 
 def upload_file(filename, overwrite=True):
+    if not overwrite and filename in EXISTING_KEYS:
+        print('Skipping existing key {}'.format(filename))
+        return
+
     print('Uploading {} to {}/{}'.format(filename, BUCKET, filename))
     ext = filename.split('.')[-1]
 
-    if not overwrite:
-        try:
-            existing = s3.get_object(Bucket=BUCKET, Key=filename)
-            print('\tSkipping existing key ', filename)
-            return
-        except:
-            pass
-
     s3.upload_file(filename, BUCKET, filename, ExtraArgs={
         'ACL': 'public-read',
         'ContentType': TYPE_MAP[ext],
@@ -140,20 +141,16 @@ def upload_bench():
     grid = filter_filenames(os.listdir('bench'), 'html')
     view = filter_filenames(os.listdir('bench/view'), 'html')
 
-    existing_keys = current_keys()
-
     for f in grid:
         upload_file('bench/{}'.format(f))
 
     for f in view:
         key = 'bench/view/{}'.format(f)
-        if key not in existing_keys:
-            upload_file('bench/view/{}'.format(f))
+        upload_file('bench/view/{}'.format(f), overwrite=False)
 
     for f in img_files:
         key = 'img/bench/{}'.format(f)
-        if key not in existing_keys:
-            upload_file(key, overwrite=False)
+        upload_file(key, overwrite=False)
 
 def upload_img():
     files = filter_filenames(os.listdir('img'), ['jpg', 'webp'])