初始化导入 new-api 源码

This commit is contained in:
OpenClaw Task Bot
2026-04-09 22:31:14 +08:00
commit 75bda2e845
994 changed files with 250292 additions and 0 deletions

View File

@@ -0,0 +1,292 @@
package gemini
import (
"bytes"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/constant"
"github.com/QuantumNous/new-api/dto"
"github.com/QuantumNous/new-api/model"
"github.com/QuantumNous/new-api/relay/channel"
taskcommon "github.com/QuantumNous/new-api/relay/channel/task/taskcommon"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/QuantumNous/new-api/service"
"github.com/QuantumNous/new-api/setting/model_setting"
"github.com/gin-gonic/gin"
"github.com/pkg/errors"
)
// ============================
// Adaptor implementation
// ============================
type TaskAdaptor struct {
taskcommon.BaseBilling
ChannelType int
apiKey string
baseURL string
}
func (a *TaskAdaptor) Init(info *relaycommon.RelayInfo) {
a.ChannelType = info.ChannelType
a.baseURL = info.ChannelBaseUrl
a.apiKey = info.ApiKey
}
// ValidateRequestAndSetAction parses body, validates fields and sets default action.
func (a *TaskAdaptor) ValidateRequestAndSetAction(c *gin.Context, info *relaycommon.RelayInfo) (taskErr *dto.TaskError) {
return relaycommon.ValidateBasicTaskRequest(c, info, constant.TaskActionTextGenerate)
}
// BuildRequestURL constructs the Gemini API predictLongRunning endpoint for Veo.
func (a *TaskAdaptor) BuildRequestURL(info *relaycommon.RelayInfo) (string, error) {
modelName := info.UpstreamModelName
version := model_setting.GetGeminiVersionSetting(modelName)
return fmt.Sprintf(
"%s/%s/models/%s:predictLongRunning",
a.baseURL,
version,
modelName,
), nil
}
// BuildRequestHeader sets required headers.
func (a *TaskAdaptor) BuildRequestHeader(c *gin.Context, req *http.Request, info *relaycommon.RelayInfo) error {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
req.Header.Set("x-goog-api-key", a.apiKey)
return nil
}
// BuildRequestBody converts request into the Veo predictLongRunning format.
func (a *TaskAdaptor) BuildRequestBody(c *gin.Context, info *relaycommon.RelayInfo) (io.Reader, error) {
v, ok := c.Get("task_request")
if !ok {
return nil, fmt.Errorf("request not found in context")
}
req, ok := v.(relaycommon.TaskSubmitReq)
if !ok {
return nil, fmt.Errorf("unexpected task_request type")
}
instance := VeoInstance{Prompt: req.Prompt}
if img := ExtractMultipartImage(c, info); img != nil {
instance.Image = img
} else if len(req.Images) > 0 {
if parsed := ParseImageInput(req.Images[0]); parsed != nil {
instance.Image = parsed
info.Action = constant.TaskActionGenerate
}
}
params := &VeoParameters{}
if err := taskcommon.UnmarshalMetadata(req.Metadata, params); err != nil {
return nil, errors.Wrap(err, "unmarshal metadata failed")
}
if params.DurationSeconds == 0 && req.Duration > 0 {
params.DurationSeconds = req.Duration
}
if params.Resolution == "" && req.Size != "" {
params.Resolution = SizeToVeoResolution(req.Size)
}
if params.AspectRatio == "" && req.Size != "" {
params.AspectRatio = SizeToVeoAspectRatio(req.Size)
}
params.Resolution = strings.ToLower(params.Resolution)
params.SampleCount = 1
body := VeoRequestPayload{
Instances: []VeoInstance{instance},
Parameters: params,
}
data, err := common.Marshal(body)
if err != nil {
return nil, err
}
return bytes.NewReader(data), nil
}
// DoRequest delegates to common helper.
func (a *TaskAdaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, requestBody io.Reader) (*http.Response, error) {
return channel.DoTaskApiRequest(a, c, info, requestBody)
}
// DoResponse handles upstream response, returns taskID etc.
func (a *TaskAdaptor) DoResponse(c *gin.Context, resp *http.Response, info *relaycommon.RelayInfo) (taskID string, taskData []byte, taskErr *dto.TaskError) {
responseBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", nil, service.TaskErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError)
}
_ = resp.Body.Close()
var s submitResponse
if err := common.Unmarshal(responseBody, &s); err != nil {
return "", nil, service.TaskErrorWrapper(err, "unmarshal_response_failed", http.StatusInternalServerError)
}
if strings.TrimSpace(s.Name) == "" {
return "", nil, service.TaskErrorWrapper(fmt.Errorf("missing operation name"), "invalid_response", http.StatusInternalServerError)
}
taskID = taskcommon.EncodeLocalTaskID(s.Name)
ov := dto.NewOpenAIVideo()
ov.ID = info.PublicTaskID
ov.TaskID = info.PublicTaskID
ov.CreatedAt = time.Now().Unix()
ov.Model = info.OriginModelName
c.JSON(http.StatusOK, ov)
return taskID, responseBody, nil
}
func (a *TaskAdaptor) GetModelList() []string {
return []string{
"veo-3.0-generate-001",
"veo-3.0-fast-generate-001",
"veo-3.1-generate-preview",
"veo-3.1-fast-generate-preview",
}
}
func (a *TaskAdaptor) GetChannelName() string {
return "gemini"
}
// EstimateBilling returns OtherRatios based on durationSeconds and resolution.
func (a *TaskAdaptor) EstimateBilling(c *gin.Context, info *relaycommon.RelayInfo) map[string]float64 {
v, ok := c.Get("task_request")
if !ok {
return nil
}
req, ok := v.(relaycommon.TaskSubmitReq)
if !ok {
return nil
}
seconds := ResolveVeoDuration(req.Metadata, req.Duration, req.Seconds)
resolution := ResolveVeoResolution(req.Metadata, req.Size)
resRatio := VeoResolutionRatio(info.UpstreamModelName, resolution)
return map[string]float64{
"seconds": float64(seconds),
"resolution": resRatio,
}
}
// FetchTask polls task status via the Gemini operations GET endpoint.
func (a *TaskAdaptor) FetchTask(baseUrl, key string, body map[string]any, proxy string) (*http.Response, error) {
taskID, ok := body["task_id"].(string)
if !ok {
return nil, fmt.Errorf("invalid task_id")
}
upstreamName, err := taskcommon.DecodeLocalTaskID(taskID)
if err != nil {
return nil, fmt.Errorf("decode task_id failed: %w", err)
}
version := model_setting.GetGeminiVersionSetting("default")
url := fmt.Sprintf("%s/%s/%s", baseUrl, version, upstreamName)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json")
req.Header.Set("x-goog-api-key", key)
client, err := service.GetHttpClientWithProxy(proxy)
if err != nil {
return nil, fmt.Errorf("new proxy http client failed: %w", err)
}
return client.Do(req)
}
func (a *TaskAdaptor) ParseTaskResult(respBody []byte) (*relaycommon.TaskInfo, error) {
var op operationResponse
if err := common.Unmarshal(respBody, &op); err != nil {
return nil, fmt.Errorf("unmarshal operation response failed: %w", err)
}
ti := &relaycommon.TaskInfo{}
if op.Error.Message != "" {
ti.Status = model.TaskStatusFailure
ti.Reason = op.Error.Message
ti.Progress = "100%"
return ti, nil
}
if !op.Done {
ti.Status = model.TaskStatusInProgress
ti.Progress = "50%"
return ti, nil
}
ti.Status = model.TaskStatusSuccess
ti.Progress = "100%"
ti.TaskID = taskcommon.EncodeLocalTaskID(op.Name)
if len(op.Response.GenerateVideoResponse.GeneratedVideos) > 0 {
if uri := op.Response.GenerateVideoResponse.GeneratedVideos[0].Video.URI; uri != "" {
ti.RemoteUrl = uri
}
}
return ti, nil
}
func (a *TaskAdaptor) ConvertToOpenAIVideo(task *model.Task) ([]byte, error) {
upstreamTaskID := task.GetUpstreamTaskID()
upstreamName, err := taskcommon.DecodeLocalTaskID(upstreamTaskID)
if err != nil {
upstreamName = ""
}
modelName := extractModelFromOperationName(upstreamName)
if strings.TrimSpace(modelName) == "" {
modelName = "veo-3.0-generate-001"
}
video := dto.NewOpenAIVideo()
video.ID = task.TaskID
video.Model = modelName
video.Status = task.Status.ToVideoStatus()
video.SetProgressStr(task.Progress)
video.CreatedAt = task.CreatedAt
if task.FinishTime > 0 {
video.CompletedAt = task.FinishTime
} else if task.UpdatedAt > 0 {
video.CompletedAt = task.UpdatedAt
}
return common.Marshal(video)
}
// ============================
// helpers
// ============================
var modelRe = regexp.MustCompile(`models/([^/]+)/operations/`)
func extractModelFromOperationName(name string) string {
if name == "" {
return ""
}
if m := modelRe.FindStringSubmatch(name); len(m) == 2 {
return m[1]
}
if idx := strings.Index(name, "models/"); idx >= 0 {
s := name[idx+len("models/"):]
if p := strings.Index(s, "/operations/"); p > 0 {
return s[:p]
}
}
return ""
}

View File

@@ -0,0 +1,138 @@
package gemini
import (
"strconv"
"strings"
)
// ParseVeoDurationSeconds extracts durationSeconds from metadata.
// Returns 8 (Veo default) when not specified or invalid.
func ParseVeoDurationSeconds(metadata map[string]any) int {
if metadata == nil {
return 8
}
v, ok := metadata["durationSeconds"]
if !ok {
return 8
}
switch n := v.(type) {
case float64:
if int(n) > 0 {
return int(n)
}
case int:
if n > 0 {
return n
}
}
return 8
}
// ParseVeoResolution extracts resolution from metadata.
// Returns "720p" when not specified.
func ParseVeoResolution(metadata map[string]any) string {
if metadata == nil {
return "720p"
}
v, ok := metadata["resolution"]
if !ok {
return "720p"
}
if s, ok := v.(string); ok && s != "" {
return strings.ToLower(s)
}
return "720p"
}
// ResolveVeoDuration returns the effective duration in seconds.
// Priority: metadata["durationSeconds"] > stdDuration > stdSeconds > default (8).
func ResolveVeoDuration(metadata map[string]any, stdDuration int, stdSeconds string) int {
if metadata != nil {
if _, exists := metadata["durationSeconds"]; exists {
if d := ParseVeoDurationSeconds(metadata); d > 0 {
return d
}
}
}
if stdDuration > 0 {
return stdDuration
}
if s, err := strconv.Atoi(stdSeconds); err == nil && s > 0 {
return s
}
return 8
}
// ResolveVeoResolution returns the effective resolution string (lowercase).
// Priority: metadata["resolution"] > SizeToVeoResolution(stdSize) > default ("720p").
func ResolveVeoResolution(metadata map[string]any, stdSize string) string {
if metadata != nil {
if _, exists := metadata["resolution"]; exists {
if r := ParseVeoResolution(metadata); r != "" {
return r
}
}
}
if stdSize != "" {
return SizeToVeoResolution(stdSize)
}
return "720p"
}
// SizeToVeoResolution converts a "WxH" size string to a Veo resolution label.
func SizeToVeoResolution(size string) string {
parts := strings.SplitN(strings.ToLower(size), "x", 2)
if len(parts) != 2 {
return "720p"
}
w, _ := strconv.Atoi(parts[0])
h, _ := strconv.Atoi(parts[1])
maxDim := w
if h > maxDim {
maxDim = h
}
if maxDim >= 3840 {
return "4k"
}
if maxDim >= 1920 {
return "1080p"
}
return "720p"
}
// SizeToVeoAspectRatio converts a "WxH" size string to a Veo aspect ratio.
func SizeToVeoAspectRatio(size string) string {
parts := strings.SplitN(strings.ToLower(size), "x", 2)
if len(parts) != 2 {
return "16:9"
}
w, _ := strconv.Atoi(parts[0])
h, _ := strconv.Atoi(parts[1])
if w <= 0 || h <= 0 {
return "16:9"
}
if h > w {
return "9:16"
}
return "16:9"
}
// VeoResolutionRatio returns the pricing multiplier for the given resolution.
// Standard resolutions (720p, 1080p) return 1.0.
// 4K returns a model-specific multiplier based on Google's official pricing.
func VeoResolutionRatio(modelName, resolution string) float64 {
if resolution != "4k" {
return 1.0
}
// 4K multipliers derived from Vertex AI official pricing (video+audio base):
// veo-3.1-generate: $0.60 / $0.40 = 1.5
// veo-3.1-fast-generate: $0.35 / $0.15 ≈ 2.333
// Veo 3.0 models do not support 4K; return 1.0 as fallback.
if strings.Contains(modelName, "3.1-fast-generate") {
return 2.333333
}
if strings.Contains(modelName, "3.1-generate") || strings.Contains(modelName, "3.1") {
return 1.5
}
return 1.0
}

View File

@@ -0,0 +1,71 @@
package gemini
// VeoImageInput represents an image input for Veo image-to-video.
// Used by both Gemini and Vertex adaptors.
type VeoImageInput struct {
BytesBase64Encoded string `json:"bytesBase64Encoded"`
MimeType string `json:"mimeType"`
}
// VeoInstance represents a single instance in the Veo predictLongRunning request.
type VeoInstance struct {
Prompt string `json:"prompt"`
Image *VeoImageInput `json:"image,omitempty"`
// TODO: support referenceImages (style/asset references, up to 3 images)
// TODO: support lastFrame (first+last frame interpolation, Veo 3.1)
}
// VeoParameters represents the parameters block for Veo predictLongRunning.
type VeoParameters struct {
SampleCount int `json:"sampleCount"`
DurationSeconds int `json:"durationSeconds,omitempty"`
AspectRatio string `json:"aspectRatio,omitempty"`
Resolution string `json:"resolution,omitempty"`
NegativePrompt string `json:"negativePrompt,omitempty"`
PersonGeneration string `json:"personGeneration,omitempty"`
StorageUri string `json:"storageUri,omitempty"`
CompressionQuality string `json:"compressionQuality,omitempty"`
ResizeMode string `json:"resizeMode,omitempty"`
Seed *int `json:"seed,omitempty"`
GenerateAudio *bool `json:"generateAudio,omitempty"`
}
// VeoRequestPayload is the top-level request body for the Veo
// predictLongRunning endpoint (used by both Gemini and Vertex).
type VeoRequestPayload struct {
Instances []VeoInstance `json:"instances"`
Parameters *VeoParameters `json:"parameters,omitempty"`
}
type submitResponse struct {
Name string `json:"name"`
}
type operationVideo struct {
MimeType string `json:"mimeType"`
BytesBase64Encoded string `json:"bytesBase64Encoded"`
Encoding string `json:"encoding"`
}
type operationResponse struct {
Name string `json:"name"`
Done bool `json:"done"`
Response struct {
Type string `json:"@type"`
RaiMediaFilteredCount int `json:"raiMediaFilteredCount"`
Videos []operationVideo `json:"videos"`
BytesBase64Encoded string `json:"bytesBase64Encoded"`
Encoding string `json:"encoding"`
Video string `json:"video"`
GenerateVideoResponse struct {
GeneratedVideos []struct {
Video struct {
URI string `json:"uri"`
} `json:"video"`
} `json:"generatedVideos"`
} `json:"generateVideoResponse"`
} `json:"response"`
Error struct {
Message string `json:"message"`
} `json:"error"`
}

View File

@@ -0,0 +1,100 @@
package gemini
import (
"encoding/base64"
"io"
"net/http"
"strings"
"github.com/QuantumNous/new-api/constant"
relaycommon "github.com/QuantumNous/new-api/relay/common"
"github.com/gin-gonic/gin"
)
const maxVeoImageSize = 20 * 1024 * 1024 // 20 MB
// ExtractMultipartImage reads the first `input_reference` file from a multipart
// form upload and returns a VeoImageInput. Returns nil if no file is present.
func ExtractMultipartImage(c *gin.Context, info *relaycommon.RelayInfo) *VeoImageInput {
mf, err := c.MultipartForm()
if err != nil {
return nil
}
files, exists := mf.File["input_reference"]
if !exists || len(files) == 0 {
return nil
}
fh := files[0]
if fh.Size > maxVeoImageSize {
return nil
}
file, err := fh.Open()
if err != nil {
return nil
}
defer file.Close()
fileBytes, err := io.ReadAll(file)
if err != nil {
return nil
}
mimeType := fh.Header.Get("Content-Type")
if mimeType == "" || mimeType == "application/octet-stream" {
mimeType = http.DetectContentType(fileBytes)
}
info.Action = constant.TaskActionGenerate
return &VeoImageInput{
BytesBase64Encoded: base64.StdEncoding.EncodeToString(fileBytes),
MimeType: mimeType,
}
}
// ParseImageInput parses an image string (data URI or raw base64) into a
// VeoImageInput. Returns nil if the input is empty or invalid.
// TODO: support downloading HTTP URL images and converting to base64
func ParseImageInput(imageStr string) *VeoImageInput {
imageStr = strings.TrimSpace(imageStr)
if imageStr == "" {
return nil
}
if strings.HasPrefix(imageStr, "data:") {
return parseDataURI(imageStr)
}
raw, err := base64.StdEncoding.DecodeString(imageStr)
if err != nil {
return nil
}
return &VeoImageInput{
BytesBase64Encoded: imageStr,
MimeType: http.DetectContentType(raw),
}
}
func parseDataURI(uri string) *VeoImageInput {
// data:image/png;base64,iVBOR...
rest := uri[len("data:"):]
idx := strings.Index(rest, ",")
if idx < 0 {
return nil
}
meta := rest[:idx]
b64 := rest[idx+1:]
if b64 == "" {
return nil
}
mimeType := "application/octet-stream"
parts := strings.SplitN(meta, ";", 2)
if len(parts) >= 1 && parts[0] != "" {
mimeType = parts[0]
}
return &VeoImageInput{
BytesBase64Encoded: b64,
MimeType: mimeType,
}
}