import { z } from 'zod'
import config from '../config'
const ComfyNodeSchema = z.object({
inputs: z.any(),
class_type: z.string(),
_meta: z.any().optional(),
})
type ComfyNode = z.infer<typeof ComfyNodeSchema>
type ComfyPrompt = Record<string, ComfyNode>
interface Workflow {
RequestSchema: z.ZodObject<any, any>
generateWorkflow: (input: any) => Promise<ComfyPrompt> | ComfyPrompt
description?: string
summary?: string
}
// This defaults the checkpoint to whatever was used in the warmup workflow
let checkpoint: any = config.models.checkpoints.enum.optional()
if (config.warmupCkpt) {
checkpoint = config.warmupCkpt
}
const RequestSchema = z.object({
prompt: z.string().describe('The prompt to generate a video clip from.'),
negative_prompt: z
.string()
.optional()
.default(
'low quality, worst quality, deformed, distorted, disfigured, motion smear, motion artifacts, fused fingers, bad anatomy, weird hand, ugly',
)
.describe('The negative prompt to generate a video clip from.'),
duration_seconds: z.number().int().min(1).max(10).default(10).describe('The duration of the video clip in seconds.'),
steps: z.number().int().min(1).max(500).default(100).describe('The number of steps to run the model for.'),
cfg: z.number().min(0).default(3.0).describe('The cfg value to use for the model.'),
seed: z
.number()
.int()
.optional()
.default(() => Math.floor(Math.random() * 100000000000))
.describe('The seed to use for the model.'),
width: z
.number()
.int()
.optional()
.default(768)
.refine((value: number) => value % 32 === 0, {
message: 'Width must be a multiple of 32.',
})
.describe('The width of the video. Must be a multiple of 32.'),
height: z
.number()
.int()
.optional()
.default(512)
.refine((value: number) => value % 32 === 0, {
message: 'Height must be a multiple of 32.',
})
.describe('The height of the video. Must be a multiple of 32.'),
})
type InputType = z.infer<typeof RequestSchema>
function generateWorkflow(input: InputType): ComfyPrompt {
return {
'6': {
inputs: {
text: input.prompt,
clip: ['38', 0],
},
class_type: 'CLIPTextEncode',
_meta: {
title: 'CLIP Text Encode (Positive Prompt)',
},
},
'7': {
inputs: {
text: input.negative_prompt,
clip: ['38', 0],
},
class_type: 'CLIPTextEncode',
_meta: {
title: 'CLIP Text Encode (Negative Prompt)',
},
},
'38': {
inputs: {
clip_name: 't5xxl_fp16.safetensors',
type: 'ltxv',
device: 'default',
},
class_type: 'CLIPLoader',
_meta: {
title: 'Load CLIP',
},
},
'44': {
inputs: {
ckpt_name: checkpoint,
},
class_type: 'CheckpointLoaderSimple',
_meta: {
title: 'Load Checkpoint',
},
},
'69': {
inputs: {
frame_rate: 24,
positive: ['6', 0],
negative: ['7', 0],
},
class_type: 'LTXVConditioning',
_meta: {
title: 'LTXVConditioning',
},
},
'70': {
inputs: {
width: input.width,
height: input.height,
length: input.duration_seconds * 24 + 1,
batch_size: 1,
},
class_type: 'EmptyLTXVLatentVideo',
_meta: {
title: 'EmptyLTXVLatentVideo',
},
},
'71': {
inputs: {
steps: input.steps,
max_shift: 2.05,
base_shift: 0.95,
stretch: true,
terminal: 0.1,
latent: ['70', 0],
},
class_type: 'LTXVScheduler',
_meta: {
title: 'LTXVScheduler',
},
},
'72': {
inputs: {
add_noise: true,
noise_seed: input.seed,
cfg: input.cfg,
model: ['44', 0],
positive: ['69', 0],
negative: ['69', 1],
sampler: ['73', 0],
sigmas: ['71', 0],
latent_image: ['70', 0],
},
class_type: 'SamplerCustom',
_meta: {
title: 'SamplerCustom',
},
},
'73': {
inputs: {
sampler_name: 'euler',
},
class_type: 'KSamplerSelect',
_meta: {
title: 'KSamplerSelect',
},
},
'77': {
inputs: {
tile_size: 512,
overlap: 64,
temporal_size: 64,
temporal_overlap: 16,
samples: ['72', 0],
vae: ['44', 2],
},
class_type: 'VAEDecodeTiled',
_meta: {
title: 'VAE Decode (Tiled)',
},
},
'78': {
inputs: {
frame_rate: 24,
loop_count: 0,
filename_prefix: 'video',
format: 'video/h265-mp4',
pix_fmt: 'yuv420p10le',
crf: 5,
save_metadata: true,
pingpong: false,
save_output: true,
images: ['77', 0],
},
class_type: 'VHS_VideoCombine',
_meta: {
title: 'Video Combine 🎥🅥🅗🅢',
},
},
}
}
const workflow: Workflow = {
RequestSchema,
generateWorkflow,
description: 'Generate a video clip from a prompt.',
summary: 'Text to Video',
}
export default workflow
