m40-llm/Cargo.toml at 0da8f39c7e879ae89563c0664dd45aeecca7d78c · GuyPaddock/m40-llm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
[package]
name = "m40-llm"
version = "0.1.0"
edition = "2021"

[lib]
name = "m40_llm"
path = "src/lib.rs"

[[bin]]
name = "m40-llm"
path = "src/main.rs"

[build-dependencies]
# `build.rs` compiles either CUDA kernels (.cu) or the non-CUDA C stub (.c).
# That requires the `cc` crate even when the `cuda` feature is disabled.
cc = "1.0"

[dependencies]
chrono = { version = "0.4", optional = true, features = ["serde", "clock"] }
rand = "0.8"
clap = { version = "4", features = ["derive"] }
tokio = { version = "1", features = ["full"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
anyhow = "1"
thiserror = "1"
bytes = "1"
reqwest = { version = "0.11", default-features = false, features = ["json", "stream", "rustls-tls"] }
tokio-stream = "0.1"
half = "2"
tiktoken = "3"

# For filesystem paths & dirs
directories = "5"

# Optional GGUF integration crates
# Keep optional and gated behind `gguf_ext` feature so default builds remain slim
# Exact versions kept flexible for now; can pin after first compile/integration pass
gguf-llms = { version = "0.0.2", optional = true }
# gguf-rs-lib provides file reader and helpers; version as discovered via crates.io
# If the exact name differs (e.g., gguf-rs), we will adjust after attempting build
gguf-rs-lib = { version = "0.2", optional = true }

[features]
logging = ["chrono"]
nvcc = []
server = ["dep:axum", "dep:tower-http"]
cuda = []  # Enable CUDA compilation support
# Enable extended GGUF integration via external crates
# Off by default to keep minimal deps for non-integration builds
gguf_ext = ["dep:gguf-llms", "dep:gguf-rs-lib"]

[dependencies.axum]
version = "0.7"
optional = true

[dependencies.tower-http]
version = "0.5"
features = ["cors", "set-header"]
optional = true


[dev-dependencies]
criterion = { version = "0.5", default-features = false }

[[bench]]
name = "gemm"
harness = false

[[bench]]
name = "gemm_fallback"
harness = false

[[bench]]
name = "q8_projection"
harness = false

[[bench]]
name = "attention"
harness = false

[[bench]]
name = "rmsnorm"
harness = false

[[bench]]
name = "stream_overlap"
harness = false

[[bench]]
name = "persistent_decode"
harness = false


[[test]]
name = "m40-llm"
path = "tests/integration_test.rs"