Skip to content

Commit 611aa7b

Browse files
Js41637Js41637
committed
Add shared git_data sync for dimmbreath repos
Introduce a centralized git_data.rs to manage cloning/pulling cached dimbreath data repos and report when downstream imports should rerun. Replaced duplicated async_process::Command-based git logic in gi, hsr and zzz with calls to git_data::sync_data_repo and added DATA_REPO_URL / DATA_DIR constants in each module. git_data supports GITHUB_DATA_PAT (injects token into remote URL), redacts tokens from errors/logs, and ensures cached repo consistency. Also exported the new git_data mod from dimbreath/mod.rs.
1 parent 74645f8 commit 611aa7b

5 files changed

Lines changed: 141 additions & 77 deletions

File tree

‎src/update/dimbreath/gi/mod.rs‎

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
use std::{
22
fs::File,
33
io::BufReader,
4-
path::Path,
54
time::{Duration, Instant},
65
};
76

@@ -12,9 +11,13 @@ mod texts;
1211
mod weapons;
1312

1413
use actix_web::rt::{self, Runtime};
15-
use async_process::Command;
1614
use sqlx::PgPool;
1715

16+
use super::git_data;
17+
18+
const DATA_REPO_URL: &str = "https://github.com/stardb-gg/genshin-data";
19+
const DATA_DIR: &str = "AnimeGameData";
20+
1821
pub async fn spawn(pool: PgPool) {
1922
std::thread::spawn(move || {
2023
let rt = Runtime::new().unwrap();
@@ -119,31 +122,7 @@ struct Configs {
119122
}
120123

121124
async fn update(up_to_date: &mut bool, pool: PgPool) -> anyhow::Result<()> {
122-
if !Path::new("dimbreath").join("AnimeGameData").exists() {
123-
Command::new("git")
124-
.args([
125-
"clone",
126-
"--depth",
127-
"1",
128-
"https://gitlab.com/Dimbreath/AnimeGameData",
129-
])
130-
.current_dir("dimbreath")
131-
.output()
132-
.await?;
133-
134-
*up_to_date = false;
135-
}
136-
137-
let output = String::from_utf8(
138-
Command::new("git")
139-
.arg("pull")
140-
.current_dir(Path::new("dimbreath").join("AnimeGameData"))
141-
.output()
142-
.await?
143-
.stdout,
144-
)?;
145-
146-
if !output.contains("Already up to date.") {
125+
if git_data::sync_data_repo(DATA_REPO_URL, DATA_DIR).await? {
147126
*up_to_date = false;
148127
}
149128

‎src/update/dimbreath/git_data.rs‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
use std::{env, fs, path::Path};
2+
3+
use anyhow::{Context as _, Result};
4+
use async_process::Command;
5+
6+
const DATA_ROOT: &str = "dimbreath";
7+
const GITHUB_DATA_PAT_ENV: &str = "GITHUB_DATA_PAT";
8+
9+
/// Syncs a cached data repo and returns true when downstream import work should rerun.
10+
pub async fn sync_data_repo(repo_url: &str, data_dir: &str) -> Result<bool> {
11+
fs::create_dir_all(DATA_ROOT)?;
12+
13+
let data_path = Path::new(DATA_ROOT).join(data_dir);
14+
let remote_url = remote_url(repo_url);
15+
let mut changed = false;
16+
17+
if data_path.exists() {
18+
match git_output(&["remote", "get-url", "origin"], &data_path).await {
19+
Ok(output) if strip_auth(output.trim()) == repo_url => {}
20+
_ => {
21+
// Existing servers may have older upstream clones cached under the same data path.
22+
fs::remove_dir_all(&data_path)?;
23+
changed = true;
24+
}
25+
}
26+
}
27+
28+
if !data_path.exists() {
29+
git_output(
30+
&["clone", "--depth", "1", &remote_url, data_dir],
31+
Path::new(DATA_ROOT),
32+
)
33+
.await?;
34+
changed = true;
35+
} else {
36+
// Store the PAT-backed remote when available so manual `git pull` works in the cache.
37+
git_output(&["remote", "set-url", "origin", &remote_url], &data_path).await?;
38+
}
39+
40+
let output = git_output(&["pull"], &data_path).await?;
41+
if !output.contains("Already up to date.") {
42+
changed = true;
43+
}
44+
45+
Ok(changed)
46+
}
47+
48+
async fn git_output(args: &[&str], current_dir: &Path) -> Result<String> {
49+
let output = git_command()
50+
.args(args)
51+
.current_dir(current_dir)
52+
.output()
53+
.await
54+
.with_context(|| {
55+
format!(
56+
"failed to start git {} in {}",
57+
redact(args.join(" ")),
58+
current_dir.display()
59+
)
60+
})?;
61+
62+
if !output.status.success() {
63+
let stderr = String::from_utf8_lossy(&output.stderr);
64+
let stdout = String::from_utf8_lossy(&output.stdout);
65+
let details = [stderr.trim(), stdout.trim()]
66+
.into_iter()
67+
.filter(|part| !part.is_empty())
68+
.collect::<Vec<_>>()
69+
.join("\n");
70+
let details = redact(details);
71+
let args = redact(args.join(" "));
72+
73+
if details.is_empty() {
74+
anyhow::bail!("git {} failed with {}", args, output.status);
75+
}
76+
77+
anyhow::bail!("git {} failed with {}: {}", args, output.status, details);
78+
}
79+
80+
Ok(String::from_utf8(output.stdout)?)
81+
}
82+
83+
fn git_command() -> Command {
84+
Command::new("git")
85+
}
86+
87+
fn remote_url(repo_url: &str) -> String {
88+
if let Ok(token) = env::var(GITHUB_DATA_PAT_ENV) {
89+
let token = token.trim();
90+
91+
if !token.is_empty() {
92+
return repo_url.replacen("https://", &format!("https://x-access-token:{token}@"), 1);
93+
}
94+
}
95+
96+
repo_url.to_string()
97+
}
98+
99+
fn strip_auth(repo_url: &str) -> String {
100+
let Some(rest) = repo_url.strip_prefix("https://") else {
101+
return repo_url.to_string();
102+
};
103+
104+
match rest.split_once('@') {
105+
Some((_, host_and_path)) => format!("https://{host_and_path}"),
106+
None => repo_url.to_string(),
107+
}
108+
}
109+
110+
fn redact(value: String) -> String {
111+
let Ok(token) = env::var(GITHUB_DATA_PAT_ENV) else {
112+
return value;
113+
};
114+
let token = token.trim();
115+
116+
if token.is_empty() {
117+
value
118+
} else {
119+
value.replace(token, "[redacted]")
120+
}
121+
}

‎src/update/dimbreath/hsr/mod.rs‎

Lines changed: 6 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
use std::{
22
fs::File,
33
io::BufReader,
4-
path::Path,
54
time::{Duration, Instant},
65
};
76

@@ -13,10 +12,14 @@ mod texts;
1312

1413
use actix_web::rt::{self, Runtime};
1514
use anyhow::Result;
16-
use async_process::Command;
1715
use serde::Deserialize;
1816
use sqlx::PgPool;
1917

18+
use super::git_data;
19+
20+
const DATA_REPO_URL: &str = "https://github.com/stardb-gg/hsr-data";
21+
const DATA_DIR: &str = "TurnBasedGameData";
22+
2023
#[derive(Deserialize)]
2124
struct AchievementData {
2225
#[serde(rename = "AchievementID")]
@@ -159,31 +162,7 @@ pub async fn spawn(pool: PgPool) {
159162
}
160163

161164
async fn update(up_to_date: &mut bool, pool: PgPool) -> Result<()> {
162-
if !Path::new("dimbreath").join("TurnBasedGameData").exists() {
163-
Command::new("git")
164-
.args([
165-
"clone",
166-
"--depth",
167-
"1",
168-
"https://gitlab.com/Dimbreath/TurnBasedGameData",
169-
])
170-
.current_dir("dimbreath")
171-
.output()
172-
.await?;
173-
174-
*up_to_date = false;
175-
}
176-
177-
let output = String::from_utf8(
178-
Command::new("git")
179-
.arg("pull")
180-
.current_dir(Path::new("dimbreath").join("TurnBasedGameData"))
181-
.output()
182-
.await?
183-
.stdout,
184-
)?;
185-
186-
if !output.contains("Already up to date.") {
165+
if git_data::sync_data_repo(DATA_REPO_URL, DATA_DIR).await? {
187166
*up_to_date = false;
188167
}
189168

‎src/update/dimbreath/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
mod git_data;
2+
13
pub mod gi;
24
pub mod hsr;
35
pub mod zzz;

‎src/update/dimbreath/zzz/mod.rs‎

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
use std::{
22
collections::HashMap,
3-
env,
43
fs::File,
54
io::BufReader,
6-
path::Path,
75
time::{Duration, Instant},
86
};
97

@@ -15,9 +13,13 @@ mod texts;
1513
mod w_engines;
1614

1715
use actix_web::rt::{self, Runtime};
18-
use async_process::Command;
1916
use sqlx::PgPool;
2017

18+
use super::git_data;
19+
20+
const DATA_REPO_URL: &str = "https://github.com/stardb-gg/zenless-data";
21+
const DATA_DIR: &str = "ZenlessData";
22+
2123
#[derive(serde::Deserialize)]
2224
struct AchieveSecondClass {
2325
#[serde(rename = "ABPBJBNNCEI")]
@@ -159,26 +161,7 @@ pub async fn spawn(pool: PgPool) {
159161
}
160162

161163
async fn update(up_to_date: &mut bool, pool: PgPool) -> anyhow::Result<()> {
162-
if !Path::new("dimbreath").join("ZenlessData").exists() {
163-
Command::new("git")
164-
.args(["clone", "--depth", "1", &env::var("ZENLESS_REPO")?])
165-
.current_dir("dimbreath")
166-
.output()
167-
.await?;
168-
169-
*up_to_date = false;
170-
}
171-
172-
let output = String::from_utf8(
173-
Command::new("git")
174-
.arg("pull")
175-
.current_dir(Path::new("dimbreath").join("ZenlessData"))
176-
.output()
177-
.await?
178-
.stdout,
179-
)?;
180-
181-
if !output.contains("Already up to date.") {
164+
if git_data::sync_data_repo(DATA_REPO_URL, DATA_DIR).await? {
182165
*up_to_date = false;
183166
}
184167

0 commit comments

Comments
 (0)