Refactor images and sounds uploads to use signed urls.

This commit is contained in:
Hikmet 2025-01-29 23:11:19 +03:00
parent 78a3e38ed0
commit 4f2d6e4b21
10 changed files with 156 additions and 215 deletions

92
package-lock.json generated
View File

@ -9,6 +9,7 @@
"version": "0.1.0",
"dependencies": {
"@aws-sdk/client-s3": "^3.735.0",
"@aws-sdk/s3-request-presigner": "^3.735.0",
"@dnd-kit/core": "^6.3.1",
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.0",
@ -27,7 +28,6 @@
"clsx": "^2.1.1",
"cmdk": "^1.0.0",
"cookies-next": "^5.1.0",
"formidable": "^3.5.2",
"get-youtube-id": "^1.0.1",
"jose": "^5.9.6",
"lucide-react": "^0.474.0",
@ -47,7 +47,6 @@
},
"devDependencies": {
"@eslint/eslintrc": "^3",
"@types/formidable": "^3.4.5",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",
@ -746,6 +745,24 @@
"node": ">=18.0.0"
}
},
"node_modules/@aws-sdk/s3-request-presigner": {
"version": "3.735.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.735.0.tgz",
"integrity": "sha512-PzfS4rWDLlp22NORWmezA8ZH6uwz7fAmYfdIbWsPKoy1Rpm+/6Kqn7Nx+Taz6UKNhGPtexutCoJqsMxCy0ZmxQ==",
"dependencies": {
"@aws-sdk/signature-v4-multi-region": "3.734.0",
"@aws-sdk/types": "3.734.0",
"@aws-sdk/util-format-url": "3.734.0",
"@smithy/middleware-endpoint": "^4.0.2",
"@smithy/protocol-http": "^5.0.1",
"@smithy/smithy-client": "^4.1.2",
"@smithy/types": "^4.1.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@aws-sdk/signature-v4-multi-region": {
"version": "3.734.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.734.0.tgz",
@ -815,6 +832,20 @@
"node": ">=18.0.0"
}
},
"node_modules/@aws-sdk/util-format-url": {
"version": "3.734.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.734.0.tgz",
"integrity": "sha512-TxZMVm8V4aR/QkW9/NhujvYpPZjUYqzLwSge5imKZbWFR806NP7RMwc5ilVuHF/bMOln/cVHkl42kATElWBvNw==",
"dependencies": {
"@aws-sdk/types": "3.734.0",
"@smithy/querystring-builder": "^4.0.1",
"@smithy/types": "^4.1.0",
"tslib": "^2.6.2"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/@aws-sdk/util-locate-window": {
"version": "3.723.0",
"resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.723.0.tgz",
@ -3787,15 +3818,6 @@
"integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==",
"dev": true
},
"node_modules/@types/formidable": {
"version": "3.4.5",
"resolved": "https://registry.npmjs.org/@types/formidable/-/formidable-3.4.5.tgz",
"integrity": "sha512-s7YPsNVfnsng5L8sKnG/Gbb2tiwwJTY1conOkJzTMRvJAlLFW1nEua+ADsJQu8N1c0oTHx9+d5nqg10WuT9gHQ==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/json-schema": {
"version": "7.0.15",
"resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@ -4348,11 +4370,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/asap": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
"integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA=="
},
"node_modules/ast-types-flow": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz",
@ -4911,15 +4928,6 @@
"resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
"integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="
},
"node_modules/dezalgo": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz",
"integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==",
"dependencies": {
"asap": "^2.0.0",
"wrappy": "1"
}
},
"node_modules/didyoumean": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz",
@ -5801,19 +5809,6 @@
"node": ">= 6"
}
},
"node_modules/formidable": {
"version": "3.5.2",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.2.tgz",
"integrity": "sha512-Jqc1btCy3QzRbJaICGwKcBfGWuLADRerLzDqi2NwSt/UkXLsHJw2TVResiaoBufHVHy9aSgClOHCeJsSsFLTbg==",
"dependencies": {
"dezalgo": "^1.0.4",
"hexoid": "^2.0.0",
"once": "^1.4.0"
},
"funding": {
"url": "https://ko-fi.com/tunnckoCore/commissions"
}
},
"node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
@ -6129,14 +6124,6 @@
"node": ">= 0.4"
}
},
"node_modules/hexoid": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/hexoid/-/hexoid-2.0.0.tgz",
"integrity": "sha512-qlspKUK7IlSQv2o+5I7yhUd7TxlOG2Vr5LTa3ve2XSNVKAL/n/u/7KLvKmFNimomDIKvZFXWHv0T12mv7rT8Aw==",
"engines": {
"node": ">=8"
}
},
"node_modules/hoist-non-react-statics": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz",
@ -7290,14 +7277,6 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"dependencies": {
"wrappy": "1"
}
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -9233,11 +9212,6 @@
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
},
"node_modules/yaml": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz",

View File

@ -10,6 +10,7 @@
},
"dependencies": {
"@aws-sdk/client-s3": "^3.735.0",
"@aws-sdk/s3-request-presigner": "^3.735.0",
"@dnd-kit/core": "^6.3.1",
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.0",
@ -28,7 +29,6 @@
"clsx": "^2.1.1",
"cmdk": "^1.0.0",
"cookies-next": "^5.1.0",
"formidable": "^3.5.2",
"get-youtube-id": "^1.0.1",
"jose": "^5.9.6",
"lucide-react": "^0.474.0",
@ -48,7 +48,6 @@
},
"devDependencies": {
"@eslint/eslintrc": "^3",
"@types/formidable": "^3.4.5",
"@types/node": "^20",
"@types/react": "^19",
"@types/react-dom": "^19",

View File

@ -4,11 +4,29 @@ import { useMutation } from "@tanstack/react-query";
export const useUploadImage = () => {
return useMutation({
mutationFn: async (image: File) => {
const formData = new FormData();
formData.append('image', image);
// First, get the pre-signed URL
const response = await apiAdmin.post('/images', null, {
headers: {
'Content-Type': image.type,
}
});
const response = await apiAdmin.post('/images', formData);
return response.data;
const { presignedUrl, url } = response.data.data;
// Then upload directly to S3
await fetch(presignedUrl, {
method: 'PUT',
body: image,
headers: {
'Content-Type': image.type,
}
});
return {
status: "success",
message: "Image uploaded successfully",
data: { url }
};
},
});
};

View File

@ -4,11 +4,29 @@ import { useMutation } from "@tanstack/react-query";
export const useUploadSound = () => {
return useMutation({
mutationFn: async (sound: File) => {
const formData = new FormData();
formData.append('sound', sound);
// First, get the pre-signed URL
const response = await apiAdmin.post('/sounds', null, {
headers: {
'Content-Type': sound.type,
}
});
const response = await apiAdmin.post('/sounds', formData);
return response.data;
const { presignedUrl, url } = response.data.data;
// Then upload directly to S3
await fetch(presignedUrl, {
method: 'PUT',
body: sound,
headers: {
'Content-Type': sound.type,
}
});
return {
status: "success",
message: "Sound uploaded successfully",
data: { url }
};
},
});
};

View File

@ -1,15 +0,0 @@
import { Files } from "formidable";
import { Fields } from "formidable";
import { IncomingForm } from "formidable";
import { NextApiRequest } from "next";
export const parseFormidableForm = async (req: NextApiRequest) => {
const form = new IncomingForm();
const [fields, files] = await new Promise<[Fields, Files]>((resolve, reject) => {
form.parse(req, (err, fields, files) => {
if (err) reject(err);
resolve([fields, files]);
});
});
return { fields, files };
};

View File

@ -1,4 +1,5 @@
import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { nanoid } from "nanoid";
export const s3Client = new S3Client({
@ -53,3 +54,24 @@ export const uploadSoundFile = async ({
return `${process.env.NEXT_PUBLIC_AUDIO_BASE_URL}/${fileName}`;
};
export const generatePresignedUrl = async (fileType: 'image' | 'audio', contentType: string) => {
const fileExtension = contentType.split('/')[1];
const fileName = `${Date.now()}-${nanoid()}.${fileExtension}`;
const key = `${fileType}s/${fileName}`; // 'images' or 'audios' folder
const putObjectCommand = new PutObjectCommand({
Bucket: process.env.CF_R2_BUCKET_NAME,
Key: key,
ContentType: contentType,
});
const presignedUrl = await getSignedUrl(s3Client, putObjectCommand, {
expiresIn: 3600, // URL expires in 1 hour
});
return {
presignedUrl,
fileUrl: `${fileType === 'image' ? process.env.NEXT_PUBLIC_IMAGE_BASE_URL : process.env.NEXT_PUBLIC_AUDIO_BASE_URL}/${fileName}`,
};
};

View File

@ -1,44 +0,0 @@
import { ApiResponse } from "@/api/api_response";
import { Files, Fields } from "formidable";
import { uploadImage as uploadImageFile } from "@/backend/lib/s3";
import fs from "fs/promises";
/**
* Handles the upload of image files.
* @param {Files} params.files - The files to upload (image)
* @param {Fields} params.fields - Additional fields if needed
*/
export const uploadImage = async (
{
files,
fields,
}: {
files: Files;
fields: Fields;
}): Promise<ApiResponse> => {
try {
const imageFile = files.image?.[0];
if (!imageFile) {
return { status: "error", message: "No image file uploaded." };
}
const imageBuffer = await fs.readFile(imageFile.filepath);
// Clean up the temp file
await fs.unlink(imageFile.filepath);
const imageUrl = await uploadImageFile({
imageBuffer,
format: 'jpg', // Assuming jpg format for simplicity
});
return {
status: "success",
message: "Image uploaded successfully.",
data: { url: imageUrl },
};
} catch (error) {
console.error("uploadImage: ", error);
return { status: "error", message: "Image upload failed." };
}
};

View File

@ -1,57 +0,0 @@
import { ApiResponse } from "@/api/api_response";
import { Files, Fields } from "formidable";
import { uploadSoundFile } from "@/backend/lib/s3";
import fs from "fs/promises";
/**
* Handles the upload of sound files.
* @param {Files} params.files - The files to upload (sound)
* @param {Fields} params.fields - Additional fields if needed
*/
export const uploadSound = async (
{
files,
fields,
}: {
files: Files;
fields: Fields;
}): Promise<ApiResponse> => {
try {
const soundFile = files.sound?.[0];
if (!soundFile) {
return { status: "error", message: "No sound file uploaded." };
}
// Determine the file format
const mimeType = soundFile.mimetype;
if (!mimeType) {
return { status: "error", message: "Could not determine the sound file format." };
}
const supportedFormats = ['audio/mpeg', 'audio/wav', 'audio/ogg'];
if (!supportedFormats.includes(mimeType)) {
return { status: "error", message: "Unsupported sound format." };
}
const format = mimeType.split('/')[1]; // Extract format from MIME type
const soundBuffer = await fs.readFile(soundFile.filepath);
// Clean up the temp file
await fs.unlink(soundFile.filepath);
const soundUrl = await uploadSoundFile({
soundBuffer,
format,
});
return {
status: "success",
message: "Sound uploaded successfully.",
data: { url: soundUrl },
};
} catch (error) {
console.error("uploadSound: ", error);
return { status: "error", message: "Sound upload failed." };
}
};

View File

@ -1,26 +1,39 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { ApiResponse } from "@/api/api_response";
import { parseFormidableForm } from "@/backend/lib/parseFormidableForm";
import { uploadImage } from "@/backend/services/uploadImage";
export const config = {
api: {
bodyParser: false, // Disable the default body parser for multipart form data
sizeLimit: '100mb',
},
};
import { generatePresignedUrl } from "@/backend/lib/s3";
export default async function handler(
req: NextApiRequest,
res: NextApiResponse<ApiResponse>
) {
if (req.method === "POST") {
const { fields, files } = await parseFormidableForm(req);
const repRes = await uploadImage({ files, fields });
res.status(200).json(repRes);
try {
const contentType = req.headers['content-type'];
if (!contentType || !contentType.startsWith('image/')) {
return res.status(400).json({
status: "error",
message: "Invalid content type. Must be an image file.",
});
}
const { presignedUrl, fileUrl } = await generatePresignedUrl('image', contentType);
res.status(200).json({
status: "success",
message: "Pre-signed URL generated successfully",
data: { presignedUrl, url: fileUrl }
});
} catch (error) {
console.error("Error generating pre-signed URL:", error);
res.status(500).json({
status: "error",
message: "Failed to generate pre-signed URL"
});
}
} else {
res
.status(405)
.json({ status: "error", message: "Unsupported request method" });
res.status(405).json({
status: "error",
message: "Unsupported request method"
});
}
}

View File

@ -1,26 +1,39 @@
import type { NextApiRequest, NextApiResponse } from "next";
import { ApiResponse } from "@/api/api_response";
import { parseFormidableForm } from "@/backend/lib/parseFormidableForm";
import { uploadSound } from "@/backend/services/uploadSound";
export const config = {
api: {
bodyParser: false, // Disable the default body parser for multipart form data
sizeLimit: '100mb',
},
};
import { generatePresignedUrl } from "@/backend/lib/s3";
export default async function handler(
req: NextApiRequest,
res: NextApiResponse<ApiResponse>
) {
if (req.method === "POST") {
const { fields, files } = await parseFormidableForm(req);
const repRes = await uploadSound({ files, fields });
res.status(200).json(repRes);
try {
const contentType = req.headers['content-type'];
if (!contentType || !contentType.startsWith('audio/')) {
return res.status(400).json({
status: "error",
message: "Invalid content type. Must be an audio file.",
});
}
const { presignedUrl, fileUrl } = await generatePresignedUrl('audio', contentType);
res.status(200).json({
status: "success",
message: "Pre-signed URL generated successfully",
data: { presignedUrl, url: fileUrl }
});
} catch (error) {
console.error("Error generating pre-signed URL:", error);
res.status(500).json({
status: "error",
message: "Failed to generate pre-signed URL"
});
}
} else {
res
.status(405)
.json({ status: "error", message: "Unsupported request method" });
res.status(405).json({
status: "error",
message: "Unsupported request method"
});
}
}