LingBot World 2 tutorial

import { NextResponse } from "next/server";

// Exchanges the server-side REACTOR_API_KEY for a short-lived session JWT.
// Keeping the exchange on the server means the API key never ships to the
// browser — the client only ever sees the JWT.
export async function POST() {
  const apiKey = process.env.REACTOR_API_KEY;
  if (!apiKey) {
    return NextResponse.json({ error: "REACTOR_API_KEY is not configured" }, { status: 500 });
  }

  const baseUrl = process.env.NEXT_PUBLIC_COORDINATOR_URL || "https://api.reactor.inc";

  const response = await fetch(`${baseUrl}/tokens`, {
    method: "POST",
    headers: { "Reactor-API-Key": apiKey, "Content-Type": "application/json" },
    body: JSON.stringify({
      authorization_details: [
        { type: "session", resources: { models: { match: ["reactor/lingbot-world-2"] } } },
      ],
    }),
  });

  if (!response.ok) {
    const body = await response.text();
    return NextResponse.json(
      { error: `Token request failed: ${response.status} ${body}` },
      { status: response.status },
    );
  }

  const { jwt } = await response.json();
  return NextResponse.json({ jwt });
}

const [jwtToken, setJwtToken] = useState<string | undefined>(undefined);

useEffect(() => {
  fetch("/api/token", { method: "POST" })
    .then(async (r) => {
      const body = await r.json().catch(() => ({}));
      if (r.ok && body.jwt) setJwtToken(body.jwt);
      else setTokenError(body.error ?? `Token request failed (${r.status})`);
    })
    .catch((err) => setTokenError(String(err)));
}, []);

/* No `autoConnect`: the user clicks Connect so they see the
   disconnected -> connecting -> waiting -> ready state machine
   first-hand. The provider owns the connection lifecycle and
   auto-disconnects on unmount, so we never call connect() from
   an effect ourselves. */
<LingbotWorld2Provider apiUrl={API_URL} jwtToken={jwtToken}>
  <StatusBar />
  <MainContent />
</LingbotWorld2Provider>;

export interface StructuredScene {
  base: LayerRegistry<string>; // world identity: subject, environment, style
  camera: LayerRegistry<ShotVariant>; // { static, dynamic } framing, selected by WASD state
  movement: LayerRegistry<ShotVariant>; // { static, dynamic } subject motion, same switch
  events: NamedEvent[]; // detail clauses bound to hold-keys 1..9
  jumpPrompt?: string; // vertical sentences for Space / C held / C release
  crouchPrompt?: string;
  standPrompt?: string;
}

export function composePrompt(
  scene: StructuredScene,
  isMoving: boolean,
  heldSlots: number[],
  verticalPrompt = "",
): string;

const recomputePromptAndSend = useCallback(() => {
  // ...pick the active vertical (jump/crouch/stand) sentence first...
  if (!sceneRef.current) return;
  const isMoving = moveLStackRef.current.length > 0 || moveLatStackRef.current.length > 0;
  const next = composePrompt(sceneRef.current, isMoving, heldSlotsRef.current, vp).trim();
  if (!next) return;
  if (next === lastSentPromptRef.current) return;
  lastSentPromptRef.current = next;
  if (isReadyRef.current) {
    lw2.setPrompt({ prompt: next }).catch(console.error);
  }
}, [sendCommand]);

const applyScene = useCallback(async (opts) => {
  // Clear any held control state so stale refs don't linger across the switch
  moveLStackRef.current = [];
  moveLatStackRef.current = [];
  // ...idle both move axes, both look directions, and all held event keys...

  // If currently generating or paused, reset first so the new scene starts clean
  if (isGenerating || isPaused) {
    lw2.reset().catch(console.error);
    // Give the backend time to process the reset before we send new data
    await new Promise((r) => setTimeout(r, 600));
  }

  // Upload the starting image and register it
  const ref = await uploadFile(file);
  await lw2.setImage({ image: ref });

  // Send the scene's composed prompt (idle state: not moving, no keys held)
  sceneRef.current = opts.scene;
  const p = composePrompt(opts.scene, false, []).trim();
  await lw2.setPrompt({ prompt: p });

  // Auto-start after a short delay to let the backend process
  await new Promise((r) => setTimeout(r, 1500));
  await lw2.start();
}, [...]);

useLingbotWorld2Message((raw) => {
  // The published schema (0.2.5) doesn't declare `workers_ready` yet, so
  // widen the union locally; every other branch narrows to its typed shape.
  const msg = raw as LingbotWorld2Message | { type: "workers_ready"; tsp_size?: number };
  if (!msg?.type) return;
  switch (msg.type) {
    case "image_accepted":
      setHasImage(true);
      setImageInfo({ w: msg.width, h: msg.height });
      break;
    case "state":
      setHasPrompt(msg.has_prompt);
      setHasImage(msg.has_image);
      setIsGenerating(msg.running && msg.started);
      setIsPaused(msg.paused);
      setCameraPoseActive(msg.camera_pose_active);
      break;
    case "chunk_complete":
      setChunkIndex(msg.chunk_index);
      // ...advance jump arcs, consume crouch dips (see below)...
      sendCameraPoseChunkRef.current(); // drive the camera-pose layer
      break;
    case "generation_reset":
      // ...clear all scene + input state; a reset ends the run...
      break;
    case "command_error":
      setErrorToast(`${msg.command || "?"}: ${msg.reason || "unknown error"}`);
      break;
  }
});

const onKeyDown = (e: KeyboardEvent) => {
  if (e.repeat) return; // re-sending the same value is a no-op anyway
  if (isTypingTarget(e.target)) return; // typing in an input must not drive the camera
  const mvL = KEY_TO_MOVE_L[e.key]; // { w: "forward", s: "back" }
  if (mvL) {
    e.preventDefault();
    const stack = moveLStackRef.current;
    if (!stack.includes(mvL)) stack.push(mvL);
    applyMovementStack();
    return;
  }
  // ...lateral axis, look, jump, crouch, event keys follow the same shape...
};

const applyMovementStack = useCallback(() => {
  pushMoveL(moveLStackRef.current.at(-1) ?? "idle"); // top of stack wins
  pushMoveLat(moveLatStackRef.current.at(-1) ?? "idle");
  // Structured prompts depend on whether anything is held; recompose so
  // base + movement[isMoving] + events stays in sync with the input state.
  if (sceneRef.current) recomputePromptAndSend();
}, [pushMoveL, pushMoveLat, recomputePromptAndSend]);

const sendCameraPoseChunk = useCallback(() => {
  const active =
    mouseLookRef.current ||
    crouchDip !== null ||
    crouchHolding ||
    rollDirRef.current !== 0 ||
    joyActive ||
    jumpMoving ||
    arrowLooking;
  if (!active) {
    if (poseSentActiveRef.current) {
      lw2.setCameraPose({ camera_pose: [] }).catch(console.error); // hand control back
      poseSentActiveRef.current = false;
    }
    return;
  }
  // Rotation + horizontal translation are UNIFORM across the chunk's latents:
  // yaw/pitch from accumulated mouse motion + held arrows, clamped; roll from Q/E.
  const ry = clampRot(
    pendingDYawRef.current * mouseSensRef.current + lookHDirRef.current * ARROW_LOOK_SPEED,
  );
  // ...rx (pitch), rz (roll), tx/tz (joystick + orbit coupling)...
  // Vertical (ty) is the only PER-LATENT component: the jump arc or crouch dip.
  const camera_pose: number[] = [];
  for (let j = 0; j < CHUNK_LATENTS; j++) {
    const intent = arcActive && pos + j < arc.length ? arc[pos + j] : 0; // +1/0/-1
    const jumpTy = arcActive ? intent * JUMP_SPEED * JUMP_UP_SIGN : uniformJumpTy;
    camera_pose.push(rx, ry, rz, tx, jumpTy + crouchTy, tz);
  }
  lw2.setCameraPose({ camera_pose }).catch(console.error);
  poseSentActiveRef.current = true;
}, [sendCommand]);

const holdPress = useCallback(
  (slot: number) => {
    const events = sceneRef.current?.events;
    if (!events || slot < 0 || slot >= events.length) return;
    if (!heldSlotsRef.current.includes(slot)) {
      heldSlotsRef.current = [...heldSlotsRef.current, slot];
    }
    setHeldSlots(heldSlotsRef.current);
    recomputePromptAndSend();
  },
  [recomputePromptAndSend],
);

const holdRelease = useCallback(
  (slot: number) => {
    if (!heldSlotsRef.current.includes(slot)) return;
    heldSlotsRef.current = heldSlotsRef.current.filter((x) => x !== slot);
    setHeldSlots(heldSlotsRef.current);
    recomputePromptAndSend();
  },
  [recomputePromptAndSend],
);

case "chunk_complete":
  // Advance the charge-mode jump arc by one chunk (CHUNK_LATENTS latents).
  // The arc auto-plays after release — no hold required during flight.
  if (jumpModeRef.current === "charge" && jumpArcRef.current.length > 0) {
    jumpArcPosRef.current += CHUNK_LATENTS;
    if (jumpArcPosRef.current >= jumpArcRef.current.length) {
      jumpArcRef.current = [];   // landed
      jumpArcPosRef.current = 0;
      recomputePromptAndSendRef.current();   // drop the jump sentence
    }
  }
  sendCameraPoseChunkRef.current();
  break;

const pushKvCacheResetMode = (mode: KvResetMode) => {
  setKvCacheResetMode(mode);
  if (isReady) sendCommand("set_kv_cache_reset", { mode }).catch(console.error);
};

case "command_error":
  setErrorToast(`${msg.command || "?"}: ${msg.reason || "unknown error"}`);
  break;