Skip to content

Commit 5215678

Browse files
DerpiusDerpius
committed
Implement range highlighting and cache tokens
1 parent b8a232b commit 5215678

5 files changed

Lines changed: 203 additions & 106 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/moonjuice-lsp/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ version = "0.1.0"
44
edition = "2024"
55

66
[dependencies]
7+
dashmap = "6.1.0"
78
moonjuice-common = { path = "../moonjuice-common", version = "0.1.2" }
89
moonjuice-lexer = { path = "../moonjuice-lexer", version = "0.1.2" }
910
moonjuice-parser = { path = "../moonjuice-parser", version = "0.1.2" }

‎packages/moonjuice-lsp/src/document.rs‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
use crate::semantic_highlighting::convert_to_lsp_tokens;
2+
use moonjuice_common::Position;
3+
use moonjuice_lexer::{Lexer, Token};
4+
use tower_lsp_server::ls_types::{
5+
Range, SemanticTokens, SemanticTokensRangeResult, SemanticTokensResult, TextDocumentContentChangeEvent,
6+
};
7+
8+
pub struct Document {
9+
tokens: Vec<Token>,
10+
}
11+
12+
impl Document {
13+
pub fn new(content: String) -> Self {
14+
Document {
15+
tokens: Lexer::tokenise(content.chars().collect()),
16+
}
17+
}
18+
19+
pub fn apply_change(&mut self, changes: Vec<TextDocumentContentChangeEvent>) {
20+
if changes.len() != 1 {
21+
return;
22+
}
23+
24+
let change = &changes[0];
25+
26+
if change.range.is_some() {
27+
return;
28+
}
29+
30+
self.tokens = Lexer::tokenise(change.text.chars().collect());
31+
}
32+
33+
pub fn get_tokens_full(&self) -> SemanticTokensResult {
34+
SemanticTokensResult::Tokens(SemanticTokens {
35+
result_id: None,
36+
data: convert_to_lsp_tokens(self.tokens.iter()),
37+
})
38+
}
39+
40+
pub fn get_tokens_range(&self, range: Range) -> SemanticTokensRangeResult {
41+
let start = Position {
42+
line: range.start.line as usize + 1,
43+
column: range.start.character as usize + 1,
44+
};
45+
let end = Position {
46+
line: range.end.line as usize + 1,
47+
column: range.end.character as usize + 1,
48+
};
49+
50+
let tokens_in_range = self.tokens.iter().filter(|token| {
51+
token.start.line >= start.line
52+
&& token.end.line <= end.line
53+
&& (token.start.line > start.line || token.start.column >= start.column)
54+
&& (token.start.line < end.line || token.start.column <= end.column)
55+
});
56+
57+
SemanticTokensRangeResult::Tokens(SemanticTokens {
58+
result_id: None,
59+
data: convert_to_lsp_tokens(tokens_in_range),
60+
})
61+
}
62+
}

‎packages/moonjuice-lsp/src/main.rs‎

Lines changed: 77 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,57 @@
1+
mod document;
12
mod semantic_highlighting;
23

3-
use crate::semantic_highlighting::SemanticHighlightingProvider;
4+
use crate::document::Document;
5+
use crate::semantic_highlighting::get_legend;
6+
use dashmap::DashMap;
47
use tower_lsp_server::ls_types::{
5-
InitializeParams, InitializeResult, InitializedParams, MessageType, SemanticTokensDeltaParams,
6-
SemanticTokensFullDeltaResult, SemanticTokensFullOptions, SemanticTokensOptions, SemanticTokensParams,
7-
SemanticTokensRangeParams, SemanticTokensRangeResult, SemanticTokensResult, SemanticTokensServerCapabilities,
8+
DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, DocumentFilter, InitializeParams,
9+
InitializeResult, InitializedParams, MessageType, SemanticTokensFullOptions, SemanticTokensOptions,
10+
SemanticTokensParams, SemanticTokensRangeParams, SemanticTokensRangeResult, SemanticTokensRegistrationOptions,
11+
SemanticTokensResult, SemanticTokensServerCapabilities, ServerCapabilities, TextDocumentRegistrationOptions,
12+
TextDocumentSyncCapability, TextDocumentSyncKind, TextDocumentSyncOptions,
813
};
914
use tower_lsp_server::{Client, LanguageServer, LspService, Server};
1015

1116
struct Backend {
1217
client: Client,
13-
semantic_highlighting_provider: SemanticHighlightingProvider,
18+
documents: DashMap<String, Document>,
1419
}
1520

1621
impl LanguageServer for Backend {
1722
async fn initialize(&self, _params: InitializeParams) -> tower_lsp_server::jsonrpc::Result<InitializeResult> {
18-
let mut result = InitializeResult::default();
19-
result.capabilities.semantic_tokens_provider = Some(SemanticTokensServerCapabilities::SemanticTokensOptions(
20-
SemanticTokensOptions {
21-
work_done_progress_options: Default::default(),
22-
legend: self.semantic_highlighting_provider.get_legend(),
23-
range: None,
24-
full: Some(SemanticTokensFullOptions::Bool(true)),
23+
let result = InitializeResult {
24+
capabilities: ServerCapabilities {
25+
text_document_sync: Some(TextDocumentSyncCapability::Options(TextDocumentSyncOptions {
26+
open_close: Some(true),
27+
change: Some(TextDocumentSyncKind::FULL),
28+
save: None,
29+
..Default::default()
30+
})),
31+
32+
semantic_tokens_provider: Some(SemanticTokensServerCapabilities::SemanticTokensRegistrationOptions(
33+
SemanticTokensRegistrationOptions {
34+
text_document_registration_options: TextDocumentRegistrationOptions {
35+
document_selector: Some(vec![DocumentFilter {
36+
language: Some("moonjuice".to_string()),
37+
scheme: Some("file".to_string()),
38+
pattern: None,
39+
}]),
40+
},
41+
semantic_tokens_options: SemanticTokensOptions {
42+
work_done_progress_options: Default::default(),
43+
legend: get_legend(),
44+
range: Some(true),
45+
full: Some(SemanticTokensFullOptions::Bool(true)),
46+
},
47+
static_registration_options: Default::default(),
48+
},
49+
)),
50+
51+
..Default::default()
2552
},
26-
));
53+
..Default::default()
54+
};
2755

2856
Ok(result)
2957
}
@@ -32,11 +60,45 @@ impl LanguageServer for Backend {
3260
self.client.log_message(MessageType::INFO, "Server initialised").await;
3361
}
3462

63+
async fn did_open(&self, params: DidOpenTextDocumentParams) {
64+
self.documents.insert(
65+
params.text_document.uri.to_string(),
66+
Document::new(params.text_document.text),
67+
);
68+
}
69+
70+
async fn did_change(&self, params: DidChangeTextDocumentParams) {
71+
if let Some(mut document) = self.documents.get_mut(params.text_document.uri.as_str()) {
72+
document.value_mut().apply_change(params.content_changes);
73+
}
74+
}
75+
76+
async fn did_close(&self, params: DidCloseTextDocumentParams) {
77+
self.documents.remove(params.text_document.uri.as_str());
78+
}
79+
3580
async fn semantic_tokens_full(
3681
&self,
3782
params: SemanticTokensParams,
3883
) -> tower_lsp_server::jsonrpc::Result<Option<SemanticTokensResult>> {
39-
self.semantic_highlighting_provider.highlight_full(params)
84+
Ok(
85+
self
86+
.documents
87+
.get(params.text_document.uri.as_str())
88+
.map(|document| document.get_tokens_full()),
89+
)
90+
}
91+
92+
async fn semantic_tokens_range(
93+
&self,
94+
params: SemanticTokensRangeParams,
95+
) -> tower_lsp_server::jsonrpc::Result<Option<SemanticTokensRangeResult>> {
96+
Ok(
97+
self
98+
.documents
99+
.get(params.text_document.uri.as_str())
100+
.map(|document| document.get_tokens_range(params.range)),
101+
)
40102
}
41103

42104
async fn shutdown(&self) -> tower_lsp_server::jsonrpc::Result<()> {
@@ -51,7 +113,7 @@ async fn main() {
51113

52114
let (service, socket) = LspService::new(|client| Backend {
53115
client,
54-
semantic_highlighting_provider: SemanticHighlightingProvider::new(),
116+
documents: Default::default(),
55117
});
56118

57119
Server::new(stdin, stdout, socket).serve(service).await;

‎packages/moonjuice-lsp/src/semantic_highlighting.rs‎

Lines changed: 62 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11
use moonjuice_common::Position;
2-
use moonjuice_lexer::{Lexer, Token, TokenValue};
3-
use std::fs;
4-
use tower_lsp_server::ls_types::{
5-
SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokens, SemanticTokensLegend, SemanticTokensParams,
6-
SemanticTokensResult,
7-
};
2+
use moonjuice_lexer::{Token, TokenValue};
3+
use tower_lsp_server::ls_types::{SemanticToken, SemanticTokenModifier, SemanticTokenType, SemanticTokensLegend};
84

95
enum TokenType {
106
Parameter = 0,
@@ -18,98 +14,73 @@ enum TokenType {
1814
Operator = 8,
1915
}
2016

21-
pub struct SemanticHighlightingProvider {}
22-
23-
impl SemanticHighlightingProvider {
24-
pub fn new() -> Self {
25-
SemanticHighlightingProvider {}
17+
fn get_token_type(token: &Token) -> Option<TokenType> {
18+
match token.value {
19+
TokenValue::Nil => Some(TokenType::Keyword),
20+
TokenValue::Bool(_) => Some(TokenType::Keyword),
21+
TokenValue::Int(_) => Some(TokenType::Number),
22+
TokenValue::Double(_) => Some(TokenType::Number),
23+
TokenValue::String(_, _) => Some(TokenType::String),
24+
TokenValue::Symbol(_) => Some(TokenType::Variable),
25+
TokenValue::Keyword(_) => Some(TokenType::Keyword),
26+
TokenValue::Operator(_) => Some(TokenType::Operator),
27+
TokenValue::SpecialCharacter(_) => None,
28+
TokenValue::Comment(_) => Some(TokenType::Comment),
29+
TokenValue::UnexpectedCharacter(_) => None,
30+
TokenValue::MalformedNumber(_) => Some(TokenType::Number),
31+
TokenValue::MalformedString(_, _) => Some(TokenType::String),
2632
}
33+
}
2734

28-
pub fn get_legend(&self) -> SemanticTokensLegend {
29-
SemanticTokensLegend {
30-
token_types: vec![
31-
SemanticTokenType::PARAMETER,
32-
SemanticTokenType::VARIABLE,
33-
SemanticTokenType::PROPERTY,
34-
SemanticTokenType::FUNCTION,
35-
SemanticTokenType::KEYWORD,
36-
SemanticTokenType::COMMENT,
37-
SemanticTokenType::STRING,
38-
SemanticTokenType::NUMBER,
39-
SemanticTokenType::OPERATOR,
40-
],
41-
token_modifiers: vec![SemanticTokenModifier::READONLY],
42-
}
35+
pub fn get_legend() -> SemanticTokensLegend {
36+
SemanticTokensLegend {
37+
token_types: vec![
38+
SemanticTokenType::PARAMETER,
39+
SemanticTokenType::VARIABLE,
40+
SemanticTokenType::PROPERTY,
41+
SemanticTokenType::FUNCTION,
42+
SemanticTokenType::KEYWORD,
43+
SemanticTokenType::COMMENT,
44+
SemanticTokenType::STRING,
45+
SemanticTokenType::NUMBER,
46+
SemanticTokenType::OPERATOR,
47+
],
48+
token_modifiers: vec![SemanticTokenModifier::READONLY],
4349
}
50+
}
4451

45-
pub fn highlight_full(
46-
&self,
47-
params: SemanticTokensParams,
48-
) -> tower_lsp_server::jsonrpc::Result<Option<SemanticTokensResult>> {
49-
if let Some(file_path) = params.text_document.uri.to_file_path() {
50-
let contents = fs::read_to_string(file_path)
51-
.map_err(|err| tower_lsp_server::jsonrpc::Error::invalid_params(err.to_string()))?;
52-
53-
let mut previous_start = Position { line: 1, column: 1 };
54-
let tokens = Lexer::tokenise(contents.chars().collect())
55-
.into_iter()
56-
.filter_map(|token| {
57-
let delta_line = if token.start.line != previous_start.line {
58-
let delta_line = token.start.line - previous_start.line;
59-
60-
previous_start = Position {
61-
line: token.start.line,
62-
column: 1,
63-
};
52+
pub fn convert_to_lsp_tokens<'a>(tokens: impl Iterator<Item = &'a Token>) -> Vec<SemanticToken> {
53+
let mut previous_start = Position { line: 1, column: 1 };
6454

65-
delta_line as u32
66-
} else {
67-
0
68-
};
55+
tokens
56+
.filter_map(|token| {
57+
let delta_line = if token.start.line != previous_start.line {
58+
let delta_line = token.start.line - previous_start.line;
6959

70-
if let Some(token_type) = Self::get_token_type(&token) {
71-
let semantic_token = SemanticToken {
72-
delta_line,
73-
delta_start: (token.start.column - previous_start.column) as u32,
74-
length: token.lexeme.len() as u32,
75-
token_type: token_type as u32,
76-
token_modifiers_bitset: 0,
77-
};
60+
previous_start = Position {
61+
line: token.start.line,
62+
column: 1,
63+
};
7864

79-
previous_start = token.start;
80-
Some(semantic_token)
81-
} else {
82-
None
83-
}
84-
})
85-
.collect();
65+
delta_line as u32
66+
} else {
67+
0
68+
};
8669

87-
Ok(Some(SemanticTokensResult::Tokens(SemanticTokens {
88-
result_id: None,
89-
data: tokens,
90-
})))
91-
} else {
92-
Err(tower_lsp_server::jsonrpc::Error::invalid_params(
93-
"Non-file text document URIs are not supported",
94-
))
95-
}
96-
}
70+
if let Some(token_type) = get_token_type(&token) {
71+
let semantic_token = SemanticToken {
72+
delta_line,
73+
delta_start: (token.start.column - previous_start.column) as u32,
74+
length: token.lexeme.len() as u32,
75+
token_type: token_type as u32,
76+
token_modifiers_bitset: 0,
77+
};
9778

98-
fn get_token_type(token: &Token) -> Option<TokenType> {
99-
match token.value {
100-
TokenValue::Nil => Some(TokenType::Keyword),
101-
TokenValue::Bool(_) => Some(TokenType::Keyword),
102-
TokenValue::Int(_) => Some(TokenType::Number),
103-
TokenValue::Double(_) => Some(TokenType::Number),
104-
TokenValue::String(_, _) => Some(TokenType::String),
105-
TokenValue::Symbol(_) => Some(TokenType::Variable),
106-
TokenValue::Keyword(_) => Some(TokenType::Keyword),
107-
TokenValue::Operator(_) => Some(TokenType::Operator),
108-
TokenValue::SpecialCharacter(_) => None,
109-
TokenValue::Comment(_) => Some(TokenType::Comment),
110-
TokenValue::UnexpectedCharacter(_) => None,
111-
TokenValue::MalformedNumber(_) => Some(TokenType::Number),
112-
TokenValue::MalformedString(_, _) => Some(TokenType::String),
113-
}
114-
}
79+
previous_start = token.start;
80+
Some(semantic_token)
81+
} else {
82+
None
83+
}
84+
})
85+
.collect()
11586
}

0 commit comments

Comments
 (0)