Add Valhalla offline files download support. (#2635)

Author: HarelM

IsraelHiking.API/Controllers/FilesController.cs

@@ -113,19 +113,20 @@ private async Task<DataContainerPoco> ConvertToDataContainer(byte[] data, string
     /// <param name="lastModified">The last time this tile was downloaded</param>
     /// <param name="tileX">The tile's X coordinates, null for root</param>
     /// <param name="tileY">The tile's Y coordinates, null for root</param>
+    /// <param name="routingTile">Whether to also list the offline routing file of the tile</param>
     /// <returns></returns>
     [HttpGet]
     [Route("offline")]
     [Authorize]
-    public async Task<IActionResult> GetOfflineFiles([FromQuery] DateTime lastModified, [FromQuery] long? tileX, [FromQuery] long? tileY)
+    public async Task<IActionResult> GetOfflineFiles([FromQuery] DateTime lastModified, [FromQuery] long? tileX, [FromQuery] long? tileY, [FromQuery] bool routingTile = false)
     {
         if (!await _receiptValidationGateway.IsEntitled(User.Identity?.Name))
         {
             _logger.LogInformation($"Unable to get the list of offline files for user: {User.Identity?.Name} since the user is not entitled, date: {lastModified}");
             return Forbid();
         }
         _logger.LogInformation($"Getting the list of offline files for user: {User.Identity?.Name}, date: {lastModified}");
-        return Ok(await _offlineFilesService.GetUpdatedFilesList(lastModified, tileX, tileY));
+        return Ok(await _offlineFilesService.GetUpdatedFilesList(lastModified, tileX, tileY, routingTile));
     }
 
     /// <summary>

IsraelHiking.API/Services/IOfflineFilesService.cs

@@ -25,6 +25,7 @@ public interface IOfflineFilesService
     /// <param name="lastModifiedDate">The date to check against</param>
     /// <param name="tileX">The tile's X coordinates, null for root</param>
     /// <param name="tileY">The tile's Y coordinates, null for root</param>
+    /// <param name="routingTile">Whether to also list the offline routing file of the tile</param>
     /// <returns>A list of file names</returns>
-    Task<Dictionary<string, DateTime>> GetUpdatedFilesList(DateTime lastModifiedDate, long? tileX, long? tileY);
+    Task<Dictionary<string, DateTime>> GetUpdatedFilesList(DateTime lastModifiedDate, long? tileX, long? tileY, bool routingTile);
 }

IsraelHiking.API/Services/OfflineFilesService.cs

@@ -58,6 +58,16 @@ public class OfflineFilesService : IOfflineFilesService
     /// <inheritdoc cref="DEM_FILE_NAME"/>
     private const string DEM_ALIAS_FILE_NAME = "raster-dem";
 
+    /// <summary>
+    /// The offline routing (valhalla) tiles of a slice, a tar file that is sliced on the fly like the other
+    /// non-DEM files. It is not a part of the style, and it is only listed for clients that ask for it since
+    /// older clients do not know how to handle it.
+    /// </summary>
+    private const string VALHALLA_FILE_NAME = "valhalla";
+
+    /// <inheritdoc cref="VALHALLA_FILE_NAME"/>
+    private const string VALHALLA_FILE_EXTENSION = ".tar";
+
     /// <summary>
     /// The fixed last modified date reported for the DEM, bumped whenever the underlying data changes.
     /// </summary>
@@ -100,7 +110,7 @@ public OfflineFilesService(IFileSystemHelper fileSystemHelper,
     /// The on-the-fly files are generated on demand, so they are always reported with today's date,
     /// while the DEM file has a fixed date and only exists at the tile level.
     /// </remarks>
-    public async Task<Dictionary<string, DateTime>> GetUpdatedFilesList(DateTime lastModifiedDate, long? tileX, long? tileY)
+    public async Task<Dictionary<string, DateTime>> GetUpdatedFilesList(DateTime lastModifiedDate, long? tileX, long? tileY, bool routingTile)
     {
         var filesDictionary = new Dictionary<string, DateTime>();
         var today = DateTime.UtcNow.Date;
@@ -117,16 +127,25 @@ public async Task<Dictionary<string, DateTime>> GetUpdatedFilesList(DateTime las
                 AddIfUpdated(filesDictionary, SourceNameToFileName(name, tileX, tileY), DEM_MODIFIED_DATE, lastModifiedDate);
             }
         }
+        if (routingTile && tileX.HasValue && tileY.HasValue)
+        {
+            AddIfUpdated(filesDictionary, SourceNameToFileName(VALHALLA_FILE_NAME, tileX, tileY, VALHALLA_FILE_EXTENSION), today, lastModifiedDate);
+        }
         return filesDictionary;
     }
 
     /// <inheritdoc/>
     /// <remarks>
     /// The DEM might be requested by its alias name, but it is stored on disk under its original name.
+    /// The routing tiles are served by their own service, everything else by the on-the-fly one.
     /// </remarks>
     public async Task<(Stream Content, long? Length)> GetFileContent(string fileName, long? tileX, long? tileY)
     {
         var sourceName = FileNameToSourceName(fileName);
+        if (sourceName == VALHALLA_FILE_NAME)
+        {
+            return await _remoteFileFetcherGateway.GetFileStream(_options.RoutingTilesAddress + fileName);
+        }
         if (!IsDem(sourceName))
         {
             return await _remoteFileFetcherGateway.GetFileStream(_options.OnTheFlyFilesAddress + fileName);
@@ -244,10 +263,10 @@ private static string FileNameToSourceName(string fileName)
         return lastDashIndex >= 0 ? name[..lastDashIndex] : name;
     }
 
-    private static string SourceNameToFileName(string sourceName, long? tileX, long? tileY)
+    private static string SourceNameToFileName(string sourceName, long? tileX, long? tileY, string extension = ".pmtiles")
     {
         return tileX.HasValue && tileY.HasValue
-            ? $"{sourceName}+{SLICE_TILE_ZOOM}-{tileX}-{tileY}.pmtiles"
-            : $"{sourceName}-{ROOT_ZOOM}.pmtiles";
+            ? $"{sourceName}+{SLICE_TILE_ZOOM}-{tileX}-{tileY}{extension}"
+            : $"{sourceName}-{ROOT_ZOOM}{extension}";
     }
 }

IsraelHiking.Common/Configuration/ConfigurationData.cs

@@ -114,6 +114,10 @@ public class ConfigurationData
     /// </summary>
     public string OnTheFlyFilesAddress { get; set; }
     /// <summary>
+    /// The base address of the service that slices and serves the offline routing (valhalla) tiles
+    /// </summary>
+    public string RoutingTilesAddress { get; set; }
+    /// <summary>
     /// OSM server base address
     /// </summary>
     public string OsmBaseAddress { get; set; }
@@ -159,6 +163,7 @@ public ConfigurationData()
         ImageCreatorServerAddress = "http://localhost:11223/";
         OfflineFilesFolder = "./";
         OnTheFlyFilesAddress = "https://mapeak.com/serve-extract/";
+        RoutingTilesAddress = "https://mapeak.com/routing-tiles/";
         OsmBaseAddress = "https://www.openstreetmap.org";
         ShareUrlApiAddress = "https://israelhiking.osm.org.il/api/urls/";
         ImageUrlsAllowList =

IsraelHiking.Web/appsettings.json

@@ -1,6 +1,7 @@
 {
   "offlineFilesFolder": "/tmp/offline",
   "onTheFlyFilesAddress": "https://mapeak.com/serve-extract/",
+  "routingTilesAddress": "https://mapeak.com/routing-tiles/",
   "logging": {
     "LogLevel": {
       "Default": "Debug",

Tests/IsraelHiking.API.Tests/Controllers/FilesControllerTests.cs

@@ -183,7 +183,7 @@ public void GetOfflineFiles_ShouldGetTheList()
     {
         _controller.SetupIdentity();
         var dict = new Dictionary<string, DateTime>();
-        _offlineFilesService.GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2)
+        _offlineFilesService.GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2, Arg.Any<bool>())
             .Returns(dict);
         _receiptValidationGateway.IsEntitled(Arg.Any<string>()).Returns(true);
 
@@ -195,6 +195,32 @@ public void GetOfflineFiles_ShouldGetTheList()
         Assert.AreEqual(dict.Count, resultDict.Count);
     }
 
+    [TestMethod]
+    public void GetOfflineFiles_OldClient_ShouldNotAskForTheRoutingTile()
+    {
+        _controller.SetupIdentity();
+        _offlineFilesService.GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2, Arg.Any<bool>())
+            .Returns(new Dictionary<string, DateTime>());
+        _receiptValidationGateway.IsEntitled(Arg.Any<string>()).Returns(true);
+
+        _ = _controller.GetOfflineFiles(DateTime.Now, 1, 2).Result;
+
+        _offlineFilesService.Received(1).GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2, false);
+    }
+
+    [TestMethod]
+    public void GetOfflineFiles_ClientAsksForTheRoutingTile_ShouldPassItOn()
+    {
+        _controller.SetupIdentity();
+        _offlineFilesService.GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2, Arg.Any<bool>())
+            .Returns(new Dictionary<string, DateTime>());
+        _receiptValidationGateway.IsEntitled(Arg.Any<string>()).Returns(true);
+
+        _ = _controller.GetOfflineFiles(DateTime.Now, 1, 2, true).Result;
+
+        _offlineFilesService.Received(1).GetUpdatedFilesList(Arg.Any<DateTime>(), 1, 2, true);
+    }
+
     [TestMethod]
     public void GetOfflineFile_NotEntitled_ShouldNotGetIt()
     {

Tests/IsraelHiking.API.Tests/Services/OfflineFilesServiceTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
 using IsraelHiking.API.Services;
@@ -18,6 +19,7 @@ namespace IsraelHiking.API.Tests.Services;
 public class OfflineFilesServiceTests
 {
     private const string OnTheFlyAddress = "https://mapeak.com/serve-extract/";
+    private const string RoutingTilesAddress = "https://mapeak.com/routing-tiles/";
     private const string StyleAddress = "https://raw.githubusercontent.com/IsraelHikingMap/VectorMap/master/Styles/mapeak-hike.json";
     private const string Style = """
     {
@@ -58,7 +60,11 @@ public void TestInitialize()
         _fileSystemHelper.CreateFileProvider(Arg.Any<string>()).Returns(_fileProvider);
         SetupStyleResponse(Style);
         var options = Substitute.For<IOptions<ConfigurationData>>();
-        options.Value.Returns(new ConfigurationData { OnTheFlyFilesAddress = OnTheFlyAddress });
+        options.Value.Returns(new ConfigurationData
+        {
+            OnTheFlyFilesAddress = OnTheFlyAddress,
+            RoutingTilesAddress = RoutingTilesAddress
+        });
         _service = new OfflineFilesService(_fileSystemHelper, _remoteFileFetcherGateway, options, Substitute.For<ILogger>());
     }
 
@@ -85,7 +91,7 @@ public void ConstructWithoutFolder_ShouldNotCreateFileProviderAgain()
     [TestMethod]
     public async Task GetUpdatedFilesList_Root_ShouldReturnOnlyOnTheFlyFilesWithTodaysDate()
     {
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null, false);
 
         Assert.HasCount(3, results);
         Assert.AreEqual(DateTime.UtcNow.Date, results["IHM-schema-6.pmtiles"]);
@@ -96,7 +102,7 @@ public async Task GetUpdatedFilesList_Root_ShouldReturnOnlyOnTheFlyFilesWithToda
     [TestMethod]
     public async Task GetUpdatedFilesList_Tile_ShouldReturnOnTheFlyAndJaxaFiles()
     {
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         Assert.HasCount(4, results);
         Assert.AreEqual(DateTime.UtcNow.Date, results["IHM-schema+7-52-75.pmtiles"]);
@@ -105,18 +111,71 @@ public async Task GetUpdatedFilesList_Tile_ShouldReturnOnTheFlyAndJaxaFiles()
         Assert.IsTrue(results.ContainsKey("jaxa_terrarium0-11_v2+7-52-75.pmtiles"));
     }
 
+    [TestMethod]
+    public async Task GetUpdatedFilesList_TileWithValhalla_ShouldAlsoReturnTheValhallaFile()
+    {
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, true);
+
+        Assert.HasCount(5, results);
+        Assert.AreEqual(DateTime.UtcNow.Date, results["valhalla+7-52-75.tar"]);
+    }
+
+    [TestMethod]
+    public async Task GetUpdatedFilesList_TileWithoutValhalla_ShouldNotReturnTheValhallaFile()
+    {
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
+
+        Assert.IsFalse(results.Keys.Any(k => k.StartsWith("valhalla")));
+    }
+
+    [TestMethod]
+    public async Task GetUpdatedFilesList_RootWithValhalla_ShouldNotReturnTheValhallaFile()
+    {
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null, true);
+
+        Assert.IsFalse(results.Keys.Any(k => k.StartsWith("valhalla")));
+    }
+
+    [TestMethod]
+    public async Task GetUpdatedFilesList_ValhallaUpToDate_ShouldNotReturnTheValhallaFile()
+    {
+        var results = await _service.GetUpdatedFilesList(DateTime.UtcNow.AddDays(1), 52, 75, true);
+
+        Assert.IsFalse(results.Keys.Any(k => k.StartsWith("valhalla")));
+    }
+
+    [TestMethod]
+    public async Task GetFileContent_ValhallaFile_ShouldBeFetchedFromTheRoutingTilesServer()
+    {
+        _remoteFileFetcherGateway.GetFileStream(Arg.Any<string>()).Returns((new MemoryStream() as Stream, (long?)0));
+
+        await _service.GetFileContent("valhalla+7-52-75.tar", 52, 75);
+
+        await _remoteFileFetcherGateway.Received(1).GetFileStream(RoutingTilesAddress + "valhalla+7-52-75.tar");
+    }
+
+    [TestMethod]
+    public async Task GetFileContent_NonValhallaFile_ShouldBeFetchedFromTheOnTheFlyServer()
+    {
+        _remoteFileFetcherGateway.GetFileStream(Arg.Any<string>()).Returns((new MemoryStream() as Stream, (long?)0));
+
+        await _service.GetFileContent("IHM-schema+7-52-75.pmtiles", 52, 75);
+
+        await _remoteFileFetcherGateway.Received(1).GetFileStream(OnTheFlyAddress + "IHM-schema+7-52-75.pmtiles");
+    }
+
     [TestMethod]
     public async Task GetUpdatedFilesList_ShouldNotReturnContourFiles()
     {
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         Assert.IsFalse(results.ContainsKey("JAXA_AW3D30_2024_contour_z5-Z12_vector+7-52-75.pmtiles"));
     }
 
     [TestMethod]
     public async Task GetUpdatedFilesList_JaxaFiles_ShouldUseFixedDates()
     {
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         Assert.AreEqual(DateTimeOffset.Parse("2026-04-09T10:36:08.8024764Z").UtcDateTime,
             results["jaxa_terrarium0-11_v2+7-52-75.pmtiles"].ToUniversalTime());
@@ -125,7 +184,7 @@ public async Task GetUpdatedFilesList_JaxaFiles_ShouldUseFixedDates()
     [TestMethod]
     public async Task GetUpdatedFilesList_EverythingUpToDate_ShouldReturnEmptyList()
     {
-        var results = await _service.GetUpdatedFilesList(DateTime.UtcNow.AddDays(1), 52, 75);
+        var results = await _service.GetUpdatedFilesList(DateTime.UtcNow.AddDays(1), 52, 75, false);
 
         Assert.IsEmpty(results);
     }
@@ -134,7 +193,7 @@ public async Task GetUpdatedFilesList_EverythingUpToDate_ShouldReturnEmptyList()
     public async Task GetUpdatedFilesList_OnlyJaxaUpToDate_ShouldReturnOnlyOnTheFlyFiles()
     {
         // A last-modified between the fixed jaxa dates and today filters out jaxa but keeps the on-the-fly files.
-        var results = await _service.GetUpdatedFilesList(new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), 52, 75);
+        var results = await _service.GetUpdatedFilesList(new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), 52, 75, false);
 
         Assert.HasCount(3, results);
         Assert.IsTrue(results.ContainsKey("IHM-schema+7-52-75.pmtiles"));
@@ -144,16 +203,16 @@ public async Task GetUpdatedFilesList_OnlyJaxaUpToDate_ShouldReturnOnlyOnTheFlyF
     [TestMethod]
     public async Task GetUpdatedFilesList_ShouldNotReadFileSystem()
     {
-        await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         _fileProvider.DidNotReceive().GetDirectoryContents(Arg.Any<string>());
     }
 
     [TestMethod]
     public async Task GetUpdatedFilesList_Twice_ShouldFetchTheStyleOnlyOnce()
     {
-        await _service.GetUpdatedFilesList(DateTime.MinValue, null, null);
-        await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        await _service.GetUpdatedFilesList(DateTime.MinValue, null, null, false);
+        await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         await _remoteFileFetcherGateway.Received(1).GetFileContent(StyleAddress);
     }
@@ -165,7 +224,7 @@ public async Task GetUpdatedFilesList_StyleFetchFailsOnce_ShouldRetryAndSucceed(
             _ => throw new Exception("some error"),
             _ => new RemoteFileFetcherGatewayResponse { Content = Encoding.UTF8.GetBytes(Style) });
 
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null, false);
 
         Assert.HasCount(3, results);
         await _remoteFileFetcherGateway.Received(2).GetFileContent(StyleAddress);
@@ -177,7 +236,7 @@ public async Task GetUpdatedFilesList_StyleFetchReturnsEmptyContent_ShouldRetryA
         _remoteFileFetcherGateway.GetFileContent(StyleAddress)
             .Returns(new RemoteFileFetcherGatewayResponse { Content = [] });
 
-        await Assert.ThrowsExactlyAsync<InvalidOperationException>(() => _service.GetUpdatedFilesList(DateTime.MinValue, null, null));
+        await Assert.ThrowsExactlyAsync<InvalidOperationException>(() => _service.GetUpdatedFilesList(DateTime.MinValue, null, null, false));
         await _remoteFileFetcherGateway.Received(3).GetFileContent(StyleAddress);
     }
 
@@ -186,7 +245,7 @@ public async Task GetUpdatedFilesList_AliasedDemSource_ShouldReturnTheAliasNameW
     {
         SetupStyleResponse(StyleWithAliasedDem);
 
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, 52, 75, false);
 
         Assert.HasCount(4, results);
         Assert.IsFalse(results.ContainsKey("jaxa_terrarium0-11_v2+7-52-75.pmtiles"));
@@ -199,7 +258,7 @@ public async Task GetUpdatedFilesList_AliasedDemSourceForRoot_ShouldNotReturnIt(
     {
         SetupStyleResponse(StyleWithAliasedDem);
 
-        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null);
+        var results = await _service.GetUpdatedFilesList(DateTime.MinValue, null, null, false);
 
         Assert.HasCount(3, results);
         Assert.IsFalse(results.ContainsKey("raster-dem-6.pmtiles"));