Auto remove expire key if paste object expired

Remove mime-types library
This commit is contained in:
Joe Ma 2022-08-19 00:59:28 +08:00
parent 825f59f00f
commit 78e6707b4b
No known key found for this signature in database
GPG key ID: 7A0ECF5F5EDC587F
3 changed files with 382 additions and 376 deletions

View file

@ -4,7 +4,6 @@
"dependencies": {
"aws4fetch": "^1.0.13",
"nanoid": "^3.3.4",
"mime-types": "^2.1.35",
"js-sha256": "^0.9.0"
},
"devDependencies": {

View file

@ -29,7 +29,7 @@
<a href="https://pb.nekoul.com">pb.nekoul.com</a> is a pastebin-like service hosted on Cloudflare Worker.<br>
This service is primarily designed for own usage and interest only.<br>
All data may be deleted or expired without any notification and guarantee. Please <b>DO NOT</b> abuse this service.<br>
The limit for file upload is <b>10 MB</b> and the paste will be kept for <b>28 days</b> only by default.<br>
The limit for file upload is <b>10 MB</b> and the paste will be kept for <b>28 days</b> only by default.<br>
The source code is available in my GitHub repository <a href="https://github.com/rikkaneko/paste">[here]</a>.<br>
This webpage is designed for upload files only.
For other operations like changing paste settings and deleting paste, please make use of the

View file

@ -18,7 +18,6 @@
import {AwsClient} from "aws4fetch";
import {customAlphabet} from "nanoid";
import {contentType} from "mime-types";
import {sha256} from "js-sha256";
// Constants
@ -27,14 +26,14 @@ const PASTE_INDEX_HTML_URL = "https://raw.githubusercontent.com/rikkaneko/paste/
const UUID_LENGTH = 4
export interface Env {
PASTE_INDEX: KVNamespace;
AWS_ACCESS_KEY_ID: string;
AWS_SECRET_ACCESS_KEY: string;
ENDPOINT: string
PASTE_INDEX: KVNamespace;
AWS_ACCESS_KEY_ID: string;
AWS_SECRET_ACCESS_KEY: string;
ENDPOINT: string
}
const API_SPEC_TEXT =
`Paste service https://${SERVICE_URL}
`Paste service https://${SERVICE_URL}
[API Specification]
GET / Fetch the Web frontpage for uploading text/file [x]
@ -82,407 +81,415 @@ const gen_id = customAlphabet(
"1234567890ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz", UUID_LENGTH);
export default {
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const {url, method, headers} = request;
const {pathname} = new URL(url);
const path = pathname.replace(/\/+$/, "") || "/";
let cache = caches.default;
async fetch(
request: Request,
env: Env,
ctx: ExecutionContext
): Promise<Response> {
const {url, method, headers} = request;
const {pathname} = new URL(url);
const path = pathname.replace(/\/+$/, "") || "/";
let cache = caches.default;
const s3 = new AwsClient({
accessKeyId: env.AWS_ACCESS_KEY_ID,
secretAccessKey: env.AWS_SECRET_ACCESS_KEY
});
const s3 = new AwsClient({
accessKeyId: env.AWS_ACCESS_KEY_ID,
secretAccessKey: env.AWS_SECRET_ACCESS_KEY
});
// Special path
if (path === "/favicon.ico" && method == "GET") {
return new Response(null, {
status: 404
// Special path
if (path === "/favicon.ico" && method == "GET") {
return new Response(null, {
status: 404
})
}
if (path === "/api" && method == "GET") {
return new Response(API_SPEC_TEXT);
}
if (path === "/") {
switch (method) {
// Fetch the HTML for uploading text/file
case "GET": {
return await fetch(PASTE_INDEX_HTML_URL, {
cf: {
cacheEverything: true
}
}).then(value => {
let res = new Response(value.body, value);
// Add the correct content-type to response header
res.headers.set("content-type", "text/html; charset=UTF-8;");
// Remove the default CSP header
res.headers.delete("content-security-policy");
return res;
})
}
// Create new paste
case "POST":
const uuid = gen_id();
let buffer: ArrayBuffer;
let title: string | undefined;
// Handle content-type
const content_type = headers.get("content-type") || "";
let mime_type: string | undefined;
let password: string | undefined;
let read_limit: number | undefined;
// Content-Type: multipart/form-data
if (content_type.includes("form")) {
const formdata = await request.formData();
const data = formdata.get("u");
if (data === null) {
return new Response("Invalid request.\n", {
status: 422
})
}
// File
if (data instanceof File) {
if (data.name) {
title = data.name;
}
buffer = await data.arrayBuffer();
// Text
} else {
buffer = new TextEncoder().encode(data)
mime_type = "text/plain; charset=UTF-8;"
}
// Set password
const pass = formdata.get("pass");
if (typeof pass === "string") {
password = pass || undefined;
}
const count = formdata.get("read-limit");
if (typeof count === "string" && !isNaN(+count)) {
read_limit = Number(count) || undefined;
}
// Raw body
} else {
if (headers.has("x-title")) {
title = headers.get("x-title") || "";
}
mime_type = headers.get("content-type") || mime_type;
password = headers.get("x-pass") || undefined;
const count = headers.get("x-read-limit") || undefined;
if (count !== undefined && !isNaN(+count)) {
read_limit = Number(count) || undefined;
}
buffer = await request.arrayBuffer();
}
// Check password rules
if (password && !check_password_rules(password)) {
return new Response("Invalid password. " +
"Password must contain alphabets and digits only, and has a length of 4 or more.", {
status: 422
})
}
}
if (path === "/api" && method == "GET") {
return new Response(API_SPEC_TEXT);
}
// Check request.body size <= 10MB
const size = buffer.byteLength;
if (size > 10485760) {
return new Response("Paste size must be under 10MB.\n", {
status: 422
});
}
if (path === "/") {
switch (method) {
// Fetch the HTML for uploading text/file
case "GET": {
return await fetch(PASTE_INDEX_HTML_URL, {
cf: {
cacheEverything: true
}
}).then(value => {
let res = new Response(value.body, value);
// Add the correct content-type to response header
res.headers.set("content-type", "text/html; charset=UTF-8;");
// Remove the default CSP header
res.headers.delete("content-security-policy");
return res;
})
}
// Check request.body size not empty
if (buffer.byteLength == 0) {
return new Response("Paste cannot be empty.\n", {
status: 422
});
}
// Create new paste
case "POST":
const uuid = gen_id();
let buffer: ArrayBuffer;
let title: string | undefined;
// Handle content-type
const content_type = headers.get("content-type") || "";
let mime_type: string | undefined;
let password: string | undefined;
let read_limit: number | undefined;
// Content-Type: multipart/form-data
if (content_type.includes("form")) {
const formdata = await request.formData();
const data = formdata.get("u");
if (data === null) {
return new Response("Invalid request.\n", {
status: 422
})
}
// File
if (data instanceof File) {
if (data.name) {
title = data.name;
mime_type = contentType(title) || undefined;
}
buffer = await data.arrayBuffer();
// Text
} else {
buffer = new TextEncoder().encode(data)
mime_type = "text/plain; charset=UTF-8;"
}
const res = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "PUT",
body: buffer
});
// Set password
const pass = formdata.get("pass");
if (typeof pass === "string") {
password = pass || undefined;
}
if (res.ok) {
// Upload success
const descriptor: PasteIndexEntry = {
title: title ?? undefined,
last_modified: Date.now(),
password: password? sha256(password).slice(0, 16): undefined,
size,
read_count_remain: read_limit,
mime_type
};
const count = formdata.get("read-limit");
if (typeof count === "string" && !isNaN(+count)) {
read_limit = Number(count) || undefined;
}
// Key will be expired after 28 day if unmodified
ctx.waitUntil(env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor), {expirationTtl: 100800}));
return new Response(get_paste_info(uuid, descriptor));
} else {
return new Response("Unable to upload the paste.\n", {
status: 500
});
}
// Raw body
} else {
if (headers.has("x-title")) {
title = headers.get("x-title") || "";
mime_type = contentType(title) || undefined;
}
mime_type = headers.get("content-type") || mime_type;
password = headers.get("x-pass") || undefined;
const count = headers.get("x-read-limit") || undefined;
if (count !== undefined && !isNaN(+count)) {
read_limit = Number(count) || undefined;
}
buffer = await request.arrayBuffer();
}
}
// Check password rules
if (password && !check_password_rules(password)) {
return new Response("Invalid password. " +
"Password must contain alphabets and digits only, and has a length of 4 or more.", {
status: 422
})
}
// Check request.body size <= 10MB
const size = buffer.byteLength;
if (size > 10485760) {
return new Response("Paste size must be under 10MB.\n", {
status: 422
});
}
// Check request.body size not empty
if (buffer.byteLength == 0) {
return new Response("Paste cannot be empty.\n", {
status: 422
});
}
const res = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "PUT",
body: buffer
});
if (res.ok) {
// Upload success
const descriptor: PasteIndexEntry = {
title: title ?? undefined,
last_modified: Date.now(),
password: password? sha256(password).slice(0, 16): undefined,
size,
read_count_remain: read_limit,
mime_type
};
ctx.waitUntil(env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor)));
return new Response(get_paste_info(uuid, descriptor));
} else {
return new Response("Unable to upload the paste.\n", {
status: 500
});
}
}
} else if (path.length >= UUID_LENGTH + 1) {
// RegExpr to match /<uuid>/<option>
const found = path.match("/(?<uuid>[A-z0-9]+)(?:/(?<option>[A-z]+))?$");
if (found === null) {
return new Response("Invalid path.\n", {
status: 403
})
}
// @ts-ignore
const {uuid, option} = found.groups;
// UUID format: [A-z0-9]{UUID_LENGTH}
if (uuid.length !== UUID_LENGTH) {
return new Response("Invalid UUID.\n", {
status: 442
})
}
const val = await env.PASTE_INDEX.get(uuid);
if (val === null) {
return new Response("Paste not found.\n", {
status: 404
});
}
const descriptor: PasteIndexEntry = JSON.parse(val);
// Handling /<uuid>/settings
if (option === "settings") {
switch(method) {
case "GET":
return new Response(get_paste_info(uuid, descriptor));
case "POST": {
// TODO Implement paste setting update
return new Response("Service is under maintainance.\n", {
status: 422
});
}
}
}
switch (method) {
// Fetch the paste by uuid
case "GET": {
// Check password if needed
if (descriptor.password !== undefined) {
if (headers.has("Authorization")) {
let cert = get_basic_auth(headers);
// Error occurred when parsing the header
if (cert === null) {
return new Response("Invalid Authorization header.", {
status: 400
});
}
// Check password and username should be empty
if (cert[0].length != 0 || descriptor.password !== sha256(cert[1]).slice(0, 16)) {
return new Response("Incorrect password.\n", {
status: 401,
headers: {
"WWW-Authenticate": "Basic realm=\"Requires password\""
}
});
}
// x-pass header
} else if (headers.has("x-pass")) {
if (descriptor.password !== sha256(headers.get("x-pass")!).slice(0, 16)) {
return new Response("Incorrect password.\n");
}
} else {
return new Response("This paste requires password.\n", {
status: 401,
headers: {
"WWW-Authenticate": "Basic realm=\"Requires password\""
}
});
}
}
// Check if access_count_remain entry present
if (descriptor.read_count_remain !== undefined) {
if (descriptor.read_count_remain <= 0) {
return new Response("Paste expired.\n", {
status: 410
});
}
descriptor.read_count_remain--;
ctx.waitUntil(env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor)));
}
// Enable CF cache for authorized request
// Match in existing cache
let res = await cache.match(request.url);
if (res === undefined) {
// Fetch form origin if not hit cache
let origin = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "GET"
});
res = new Response(origin.body, origin);
if (!res.ok) {
// UUID exists in index but not found in remote object storage service
return new Response("Internal server error.\n", {
status: 500
});
}
// Remove x-amz-* headers
for (let [key, value] of res.headers.entries()) {
if (key.startsWith("x-amz")) {
res.headers.delete(key);
}
}
res.headers.set("cache-control", "public, max-age=18000");
// Alter content type to text/plain
if (option === "raw") {
res.headers.set("content-type", "text/plain; charset=UTF-8;");
} else {
res.headers.set("content-type", descriptor.mime_type ?? "application/octet-stream");
}
res.headers.set("content-disposition",
`inline; filename="${encodeURIComponent(descriptor.title ?? uuid)}"`);
if (option === "download") {
res.headers.set("content-disposition",
`attachment; filename="${encodeURIComponent(descriptor.title ?? uuid)}"`);
}
// res.body cannot be read twice
// Do not block when writing to cache
ctx.waitUntil(cache.put(url, res.clone()));
return res;
}
// Cache hit
let { readable, writable } = new TransformStream();
res.body!.pipeTo(writable);
return new Response(readable, res);
}
// Delete paste by uuid
case "DELETE": {
if (descriptor.editable !== undefined && !descriptor.editable) {
return new Response("This paste is immutable.\n", {
status: 405
});
}
// Check password if needed
if (descriptor.password !== undefined) {
if (headers.has("x-pass")) {
const pass = headers.get("x-pass");
if (descriptor.password !== sha256(pass!).slice(0, 16)) {
return new Response("Incorrect password.\n", {
status: 403
});
}
} else {
return new Response("This operation requires password.\n", {
status: 401
})
}
}
let res = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "DELETE"
});
if (res.ok) {
ctx.waitUntil(env.PASTE_INDEX.delete(uuid));
// Invalidate CF cache
ctx.waitUntil(cache.delete(url));
return new Response("OK\n");
} else {
return new Response("Unable to process such request.\n", {
status: 500
});
}
}
}
}
// Default response
} else if (path.length >= UUID_LENGTH + 1) {
// RegExpr to match /<uuid>/<option>
const found = path.match("/(?<uuid>[A-z0-9]+)(?:/(?<option>[A-z]+))?$");
if (found === null) {
return new Response("Invalid path.\n", {
status: 403
status: 403
})
}
// @ts-ignore
const {uuid, option} = found.groups;
// UUID format: [A-z0-9]{UUID_LENGTH}
if (uuid.length !== UUID_LENGTH) {
return new Response("Invalid UUID.\n", {
status: 442
})
}
const val = await env.PASTE_INDEX.get(uuid);
if (val === null) {
return new Response("Paste not found.\n", {
status: 404
});
},
}
const descriptor: PasteIndexEntry = JSON.parse(val);
// Handling /<uuid>/settings
if (option === "settings") {
switch(method) {
case "GET":
return new Response(get_paste_info(uuid, descriptor));
case "POST": {
// TODO Implement paste setting update
return new Response("Service is under maintainance.\n", {
status: 422
});
}
}
}
switch (method) {
// Fetch the paste by uuid
case "GET": {
// Check password if needed
if (descriptor.password !== undefined) {
if (headers.has("Authorization")) {
let cert = get_basic_auth(headers);
// Error occurred when parsing the header
if (cert === null) {
return new Response("Invalid Authorization header.", {
status: 400
});
}
// Check password and username should be empty
if (cert[0].length != 0 || descriptor.password !== sha256(cert[1]).slice(0, 16)) {
return new Response("Incorrect password.\n", {
status: 401,
headers: {
"WWW-Authenticate": "Basic realm=\"Requires password\""
}
});
}
// x-pass header
} else if (headers.has("x-pass")) {
if (descriptor.password !== sha256(headers.get("x-pass")!).slice(0, 16)) {
return new Response("Incorrect password.\n");
}
} else {
return new Response("This paste requires password.\n", {
status: 401,
headers: {
"WWW-Authenticate": "Basic realm=\"Requires password\""
}
});
}
}
// Check if access_count_remain entry present
if (descriptor.read_count_remain !== undefined) {
if (descriptor.read_count_remain <= 0) {
return new Response("Paste expired.\n", {
status: 410
});
}
descriptor.read_count_remain--;
ctx.waitUntil(env.PASTE_INDEX.put(uuid, JSON.stringify(descriptor), {expirationTtl: 100800}));
}
// Enable CF cache for authorized request
// Match in existing cache
let res = await cache.match(request.url);
if (res === undefined) {
// Fetch form origin if not hit cache
let origin = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "GET"
});
res = new Response(origin.body, origin);
if (res.status == 404) {
// UUID exists in index but not found in remote object storage service, probably expired
// Remove expired key
ctx.waitUntil(env.PASTE_INDEX.delete(uuid));
// Invalidate CF cache
ctx.waitUntil(cache.delete(url));
return new Response("Paste expired.\n", {
status: 410
});
} else if (!res.ok) {
// Other error
return new Response("Internal server error.\n", {
status: 500
});
}
// Remove x-amz-* headers
for (let [key, value] of res.headers.entries()) {
if (key.startsWith("x-amz")) {
res.headers.delete(key);
}
}
res.headers.set("cache-control", "public, max-age=18000");
// Alter content type to text/plain
if (option === "raw" || descriptor.mime_type === undefined) {
res.headers.delete("content-type");
} else {
res.headers.set("content-type", descriptor.mime_type);
}
res.headers.set("content-disposition",
`inline; filename="${encodeURIComponent(descriptor.title ?? uuid)}"`);
if (option === "download") {
res.headers.set("content-disposition",
`attachment; filename="${encodeURIComponent(descriptor.title ?? uuid)}"`);
}
// res.body cannot be read twice
// Do not block when writing to cache
ctx.waitUntil(cache.put(url, res.clone()));
return res;
}
// Cache hit
let { readable, writable } = new TransformStream();
res.body!.pipeTo(writable);
return new Response(readable, res);
}
// Delete paste by uuid
case "DELETE": {
if (descriptor.editable !== undefined && !descriptor.editable) {
return new Response("This paste is immutable.\n", {
status: 405
});
}
// Check password if needed
if (descriptor.password !== undefined) {
if (headers.has("x-pass")) {
const pass = headers.get("x-pass");
if (descriptor.password !== sha256(pass!).slice(0, 16)) {
return new Response("Incorrect password.\n", {
status: 403
});
}
} else {
return new Response("This operation requires password.\n", {
status: 401
})
}
}
let res = await s3.fetch(`${env.ENDPOINT}/${uuid}`, {
method: "DELETE"
});
if (res.ok) {
ctx.waitUntil(env.PASTE_INDEX.delete(uuid));
// Invalidate CF cache
ctx.waitUntil(cache.delete(url));
return new Response("OK\n");
} else {
return new Response("Unable to process such request.\n", {
status: 500
});
}
}
}
}
// Default response
return new Response("Invalid path.\n", {
status: 403
});
},
};
function get_paste_info(uuid: string, descriptor: PasteIndexEntry): string {
const date = new Date(descriptor.last_modified)
return `link: https://${SERVICE_URL}/${uuid}
id: ${uuid}
const date = new Date(descriptor.last_modified)
return`id: ${uuid}
link: https://${SERVICE_URL}/${uuid}
title: ${descriptor.title || "<empty>"}
mime-type: ${descriptor.mime_type ?? "application/octet-stream"}
mime-type: ${descriptor.mime_type ?? "-"}
size: ${descriptor.size} bytes (${to_human_readable_size(descriptor.size)})
password: ${(!!descriptor.password)}
editable: ${descriptor.editable? descriptor.editable: true}
remaining read count: ${descriptor.read_count_remain !== undefined?
descriptor.read_count_remain? descriptor.read_count_remain: `0 (expired)`: "-"}
remaining read count: ${descriptor.read_count_remain !== undefined?
descriptor.read_count_remain? descriptor.read_count_remain: `0 (expired)`: "-"}
created at ${date.toISOString()}
`
}
function check_password_rules(password: string): boolean {
return password.match("^[A-z0-9]{4,}$") !== null;
return password.match("^[A-z0-9]{4,}$") !== null;
}
// Extract username and password from Basic Authorization header
function get_basic_auth(headers: Headers): [string, string] | null {
if (headers.has("Authorization")) {
const auth = headers.get("Authorization");
const [scheme, encoded] = auth!.split(" ");
// Validate authorization header format
if (!encoded || scheme !== "Basic") {
return null;
}
// Decode base64 to string (UTF-8)
const buffer = Uint8Array.from(atob(encoded), character => character.charCodeAt(0));
const decoded = new TextDecoder().decode(buffer).normalize();
const index = decoded.indexOf(':');
// Check if user & password are split by the first colon and MUST NOT contain control characters.
if (index === -1 || decoded.match("[\\0-\x1F\x7F]")) {
return null;
}
return [decoded.slice(0, index), decoded.slice(index + 1)];
} else {
return null;
if (headers.has("Authorization")) {
const auth = headers.get("Authorization");
const [scheme, encoded] = auth!.split(" ");
// Validate authorization header format
if (!encoded || scheme !== "Basic") {
return null;
}
// Decode base64 to string (UTF-8)
const buffer = Uint8Array.from(atob(encoded), character => character.charCodeAt(0));
const decoded = new TextDecoder().decode(buffer).normalize();
const index = decoded.indexOf(':');
// Check if user & password are split by the first colon and MUST NOT contain control characters.
if (index === -1 || decoded.match("[\\0-\x1F\x7F]")) {
return null;
}
return [decoded.slice(0, index), decoded.slice(index + 1)];
} else {
return null;
}
}
function to_human_readable_size(bytes: number): string {
let size = bytes + " bytes";
const units = ["KiB", "MiB", "GiB", "TiB"];
for (let i = 0, approx = bytes / 1024; approx > 1; approx /= 1024, i++) {
size = approx.toFixed(3) + " " + units[i];
}
return size;
let size = bytes + " bytes";
const units = ["KiB", "MiB", "GiB", "TiB"];
for (let i = 0, approx = bytes / 1024; approx > 1; approx /= 1024, i++) {
size = approx.toFixed(3) + " " + units[i];
}
return size;
}
interface PasteIndexEntry {
title?: string,
mime_type?: string,
last_modified: number,
size: number,
password?: string,
editable?: boolean, // Default: True
read_count_remain?: number
title?: string,
mime_type?: string,
last_modified: number,
size: number,
password?: string,
editable?: boolean, // Default: True
read_count_remain?: number
}