Use request forwarding for large paste

Signed-off-by: Joe Ma <rikkaneko23@gmail.com>

src/index.ts

@@ -19,7 +19,7 @@
 import { AwsClient } from 'aws4fetch';
 import { sha256 } from 'js-sha256';
 import { Router, error } from 'itty-router';
-import { ERequest, Env, PasteIndexEntry, PASTE_TYPES } from './types';
+import { ERequest, Env, PasteIndexEntry } from './types';
 import { serve_static } from './proxy';
 import { check_password_rules, get_paste_info, get_basic_auth, gen_id } from './utils';
 import { UUID_LENGTH, PASTE_WEB_URL, SERVICE_URL, CORS_DOMAIN } from './constant';
@@ -334,6 +334,7 @@ router.get('/:uuid/:option?', async (request, env, ctx) => {
 
   // New added in 2.0
   // Handle large_paste
+  // Use presigned url generation only if the file size larger than 200MB, use request forwarding instead
   if (descriptor.type === 'large_paste') {
     if (!descriptor.upload_completed) {
       return new Response('This paste is not yet finalized.\n', {
@@ -341,21 +342,23 @@ router.get('/:uuid/:option?', async (request, env, ctx) => {
       });
     }
 
-    const signed_url = await get_presign_url(uuid, descriptor, env);
+    if (descriptor.size >= 209715200) {
+      const signed_url = await get_presign_url(uuid, descriptor, env);
 
-    ctx.waitUntil(
+      ctx.waitUntil(
-      env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor), {
+        env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor), {
-        expiration: descriptor.expiration! / 1000,
+          expiration: descriptor.expiration! / 1000,
-      })
+        })
-    );
+      );
 
-    return new Response(null, {
+      return new Response(null, {
-      status: 301,
+        status: 301,
-      headers: {
+        headers: {
-        location: signed_url,
+          location: signed_url,
-        'cache-control': 'no-store',
+          'cache-control': 'no-store',
-      },
+        },
-    });
+      });
+    }
   }
 
   // Enable CF cache for authorized request
@@ -375,13 +378,20 @@ router.get('/:uuid/:option?', async (request, env, ctx) => {
 
   let res = await cache.match(req_key);
   if (res === undefined) {
+    // Use althernative endpoint and credentials for large_type
+    const endpoint = descriptor.type === 'large_paste' ? env.LARGE_DOWNLOAD_ENDPOINT : env.ENDPOINT;
+    const access_key_id = descriptor.type === 'large_paste' ? env.LARGE_AWS_ACCESS_KEY_ID! : env.AWS_ACCESS_KEY_ID;
+    const secret_access_key =
+      descriptor.type === 'large_paste' ? env.LARGE_AWS_SECRET_ACCESS_KEY! : env.AWS_SECRET_ACCESS_KEY;
+
     const s3 = new AwsClient({
-      accessKeyId: env.AWS_ACCESS_KEY_ID,
+      accessKeyId: access_key_id,
-      secretAccessKey: env.AWS_SECRET_ACCESS_KEY,
+      secretAccessKey: secret_access_key,
       service: 's3', // required
     });
+
     // Fetch form origin if not hit cache
-    let origin = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
+    let origin = await s3.fetch(`${endpoint}/${uuid}`, {
       method: 'GET',
       headers: match_etag
         ? {
@@ -527,12 +537,19 @@ router.delete('/:uuid', async (request, env, ctx) => {
       });
     }
   }
+
+  // Use althernative endpoint and credentials for large_type
   const endpoint = descriptor.type === 'large_paste' ? env.LARGE_DOWNLOAD_ENDPOINT : env.ENDPOINT;
+  const access_key_id = descriptor.type === 'large_paste' ? env.LARGE_AWS_ACCESS_KEY_ID! : env.AWS_ACCESS_KEY_ID;
+  const secret_access_key =
+    descriptor.type === 'large_paste' ? env.LARGE_AWS_SECRET_ACCESS_KEY! : env.AWS_SECRET_ACCESS_KEY;
+
   const s3 = new AwsClient({
-    accessKeyId: descriptor.type === 'large_paste' ? env.LARGE_AWS_ACCESS_KEY_ID! : env.AWS_ACCESS_KEY_ID,
+    accessKeyId: access_key_id,
-    secretAccessKey: descriptor.type === 'large_paste' ? env.LARGE_AWS_SECRET_ACCESS_KEY! : env.AWS_SECRET_ACCESS_KEY,
+    secretAccessKey: secret_access_key,
     service: 's3', // required
   });
+
   let res = await s3.fetch(`${endpoint}/${uuid}`, {
     method: 'DELETE',
   });

src/types.d.ts

@@ -6,7 +6,7 @@ export type ERequest = {
   // match_etag?: string;
 } & IRequest;
 
-export type PASTE_TYPES = 'paste' | 'link' | 'large_paste';
+export type PASTE_TYPES = 'paste' | 'text' | 'link' | 'large_paste';
 
 export interface PasteIndexEntry {
   title?: string;
@@ -20,7 +20,6 @@ export interface PasteIndexEntry {
   type: PASTE_TYPES;
   // Only apply when large_paste
   upload_completed?: boolean;
-  sha256_hash?: string;
   cached_presigned_url?: string;
   cached_presigned_url_expiration?: number;
 }

src/utils.ts

@@ -23,7 +23,7 @@ import { PasteIndexEntry, Env } from './types';
 
 export const gen_id = customAlphabet('1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz', UUID_LENGTH);
 
-export function get_paste_info_obj(uuid: string, descriptor: PasteIndexEntry, env: Env) {
+export function get_paste_info_obj(uuid: string, descriptor: PasteIndexEntry) {
   const created = new Date(descriptor.last_modified);
   const expired = new Date(descriptor.expiration ?? descriptor.last_modified + 2419200000);
   const link = `https://${SERVICE_URL}/${uuid}`;
@@ -53,7 +53,7 @@ export async function get_paste_info(
   need_qr: boolean = false,
   reply_json = false
 ): Promise<Response> {
-  const paste_info = get_paste_info_obj(uuid, descriptor, env);
+  const paste_info = get_paste_info_obj(uuid, descriptor);
 
   // Reply with JSON
   if (reply_json) {
@@ -167,6 +167,7 @@ export function get_basic_auth(headers: Headers): [string, string] | null {
     return null;
   }
 }
+
 function to_human_readable_size(bytes: number): string {
   let size = bytes + ' bytes';
   const units = ['KiB', 'MiB', 'GiB', 'TiB'];

src/v2/large_upload.ts

@@ -165,7 +165,6 @@ router.post('/create', async (request, env, ctx) => {
     type: 'large_paste',
     size: file_size,
     upload_completed: false,
-    sha256_hash: file_hash,
   };
 
   ctx.waitUntil(
@@ -250,7 +249,7 @@ router.post('/complete/:uuid', async (request, env, ctx) => {
   const paste_info = {
     upload_completed: true,
     expired: new Date(expriation).toISOString(),
-    paste_info: get_paste_info_obj(uuid, descriptor, env),
+    paste_info: get_paste_info_obj(uuid, descriptor),
   };
 
   return new Response(JSON.stringify(paste_info));