# Public HEAT API tutorial: credential-free distribution copy # Changing location requires country-specific population, mortality, pollution, crash and valuation inputs; changing IDs alone is insufficient. # HEAT API tutorial: run multiple assessments from an exported HEAT template # # Audience: first-time HEAT API users who want to automate many runs in R. # This script is a cleaned-up example based on a working user workflow. # Guidance and codebooks can be found here: https://api.heatwalkingcycling.org/apiv1/readme.html # General information about HEAT, and the tool itself are here: https://www.heatwalkingcycling.org # IMPORTANT BEFORE YOU START # 1) Create and run ONE initial assessment in the HEAT web tool first. # www.heatwalkingcycling.org/tool/ # 2) Make sure that assessment is close to your planned batch runs: # - same transport modes (walking / cycling / etc.) # - same pathways/benefits you want included # - same overall structure # This matters because the exported `.rds` template contains the fields that # the API expects. If a field is missing from the template, your loop cannot # update it later. # 3) Export the HEAT webapp input `.rds` file from the web tool. # 4) If you will run other locations than the template location, prepare valid # HEAT `location_id` and `country_location_id` values for those locations. # # Notes: # - HEAT API rate limit is 5 calls per minute. This script waits 13 seconds # between calls to stay below the limit. # - The API does not fully validate your inputs. Check a sample of results. # ------------------------------------ # # Step 1. Install/load packages ---- # ------------------------------------ # # Uncomment if needed: # install.packages(c("httr", "jsonlite", "dplyr", "readr", "openxlsx", "tibble")) library(httr) library(jsonlite) library(dplyr) library(readr) library(openxlsx) library(tibble) # Helper: drop unnamed columns created by CSV exports (readr names them `...1`, `...2`, ...) drop_unnamed_csv_columns <- function(df, table_name = "data") { unnamed_cols <- names(df)[grepl("^\\.\\.\\.[0-9]+$", names(df))] if (length(unnamed_cols) == 0) { return(df) } message( "Dropping unnamed column(s) from ", table_name, ": ", paste(unnamed_cols, collapse = ", "), " (these usually come from an empty header column in the CSV)." ) dplyr::select(df, -all_of(unnamed_cols)) } # Helper: normalize HEAT ID columns so `coalesce()` does not fail on type mismatch # Converts blanks/"NA"/"NULL" strings to NA, then parses numeric IDs. normalize_heat_id <- function(x) { x_chr <- trimws(as.character(x)) x_chr[x_chr %in% c("", "NA", "NaN", "NULL", "null")] <- NA_character_ suppressWarnings(as.numeric(x_chr)) } # --------------------------------------------------- # # Step 2. Set paths, API cookie, and preferences ---- # --------------------------------------------------- # # Change this folder to where your files are stored project_dir <- getwd() # folder containing your export and scenarios.csv # Files you provide template_rds_path <- file.path(project_dir, "webapp_inputs.rds") # Scenario CSV import settings (adjust for local exports) # Common defaults: comma-delimited UTF-8 # Example (many Colombia/Excel exports): semicolon + Latin1 or Windows-1252 # scenario_csv_delim <- ";" # scenario_csv_locale <- readr::locale(encoding = "Latin1", decimal_mark = ",", grouping_mark = ".") scenario_csv_path <- file.path(project_dir, "scenarios.csv") scenario_csv_delim <- "," scenario_csv_locale <- readr::locale(encoding = "UTF-8") # Optional: a lookup table with HEAT location IDs (can be country-specific) # Set to NULL if your scenario CSV already contains `ucc_location_id` and # `ucc_country_location_id`, or if you are not changing location. location_lookup_csv_path <- NULL location_lookup_csv_delim <- "," location_lookup_csv_locale <- readr::locale(encoding = "UTF-8") # Output files created by this script output_xlsx_path <- file.path(project_dir, "HEAT_API_multi_assessment_results.xlsx") output_rds_path <- file.path(project_dir, "HEAT_API_multi_assessment_results.rds") # HEAT API endpoint heat_api_url <- "https://api.heatwalkingcycling.org/apiv1/heat_5_0/results?output_format=json" # HEAT API session cookie (user cookie) # Recommended: store it in an environment variable instead of hard-coding here. #----------------------# # Example in R session: # Supply your own cookie through the HEAT_API_USER_COOKIE environment variable. # Never put a real cookie in shared code. #----------------------# cookie_value <- Sys.getenv("HEAT_API_USER_COOKIE") if (!nzchar(cookie_value)) { stop("Set HEAT_API_USER_COOKIE privately in your environment before running this script. Request access from heatwalkingcycling@who.int. Never share your cookie.") } # Column in your scenario CSV used to identify each run in the outputs # (e.g., city name, scenario ID, year, etc.) scenario_label_column <- "run_id" # Optional: use a column from your scenario CSV to join to location lookup table. # In many cases this is the same as `scenario_label_column`. location_lookup_name_column <- "run_id" # Pause between API calls (HEAT API limit is 5 calls/minute) wait_seconds_between_calls <- 13 # Optional filtering for output results: # Set to NULL to keep all pathways, or set a character vector like: # c("Physical activity") keep_pathways <- NULL # --------------------------------------------- # # Step 3. Load the exported HEAT template ---- # --------------------------------------------- # if (!file.exists(template_rds_path)) { stop("Template .rds file not found: ", template_rds_path) } template_webapp_input <- readRDS(template_rds_path) if (!is.list(template_webapp_input)) { stop("The template .rds file does not contain a list-like `webapp_input` object.") } message("Loaded template with ", length(template_webapp_input), " fields.") # Helper: the following parameters are available for manipulation writeLines(sort(names(template_webapp_input)), con = file.path(project_dir, "template_field_names.txt")) # This creates a list of fields you can use in your scenario CSV. # The code book for these is here: # For "looped" parameters (mode, pathway, or case-specific) # https://api.heatwalkingcycling.org/apiv1/HEAT_codebook_UI_inputvars_loops_expanded.csv # For "global" parameters (default values, etc.) # https://api.heatwalkingcycling.org/apiv1/HEAT_codebook_default_background_parameters.csv # --------------------------------------------- # # Step 4. Load your scenario input table ---- # --------------------------------------------- # if (!file.exists(scenario_csv_path)) { stop("Scenario CSV file not found: ", scenario_csv_path) } scenarios <- readr::read_delim( file = scenario_csv_path, delim = scenario_csv_delim, locale = scenario_csv_locale, show_col_types = FALSE ) scenarios <- drop_unnamed_csv_columns(scenarios, "scenario CSV") if (!(scenario_label_column %in% names(scenarios))) { stop("Scenario label column not found in scenario CSV: `", scenario_label_column, "`") } message("Loaded ", nrow(scenarios), " scenarios.") # --------------------------------------------------------------- # # Step 5. (Optional) Add HEAT location IDs via lookup table ---- # --------------------------------------------------------------- # # Why this matters: # If you change location, HEAT needs valid internal IDs. These are usually # `ucc_location_id` and `ucc_country_location_id` in the webapp input. # incl. advanced options, available geographic/administrative hierarchies are: # "country" "sub1" "sub2" "city" # NOTE: background data may not necessarily be available at all levels. If # missing will default to next higher level. if (!is.null(location_lookup_csv_path)) { if (!file.exists(location_lookup_csv_path)) { stop("Location lookup CSV not found: ", location_lookup_csv_path) } location_lookup <- readr::read_delim( file = location_lookup_csv_path, delim = location_lookup_csv_delim, locale = location_lookup_csv_locale, show_col_types = FALSE ) location_lookup <- drop_unnamed_csv_columns(location_lookup, "location lookup CSV") required_lookup_cols <- c("location_id", "country_location_id") if (!all(required_lookup_cols %in% names(location_lookup))) { stop( "Location lookup file must contain columns: ", paste(required_lookup_cols, collapse = ", ") ) } # Try a common HEAT city-name column automatically, otherwise ask user to adapt. candidate_name_cols <- c("city_location_name", "location_name") lookup_name_col <- candidate_name_cols[candidate_name_cols %in% names(location_lookup)][1] if (is.na(lookup_name_col)) { stop( "Could not find a location name column in lookup file. ", "Expected one of: ", paste(candidate_name_cols, collapse = ", "), ". Please adapt this section to your file." ) } if (!(location_lookup_name_column %in% names(scenarios))) { stop( "Scenario column used for location lookup not found: `", location_lookup_name_column, "`" ) } # Join HEAT IDs to the scenarios (rename lookup columns to avoid collisions) scenarios <- scenarios %>% left_join( location_lookup %>% select( all_of(lookup_name_col), location_id, country_location_id ) %>% rename( lookup_location_id = location_id, lookup_country_location_id = country_location_id ), by = setNames(lookup_name_col, location_lookup_name_column) ) # Ensure target HEAT ID columns exist, then fill missing values from lookup if (!("ucc_location_id" %in% names(scenarios))) { scenarios$ucc_location_id <- NA_real_ } if (!("ucc_country_location_id" %in% names(scenarios))) { scenarios$ucc_country_location_id <- NA_real_ } scenarios <- scenarios %>% mutate( ucc_location_id = normalize_heat_id(ucc_location_id), ucc_country_location_id = normalize_heat_id(ucc_country_location_id), lookup_location_id = normalize_heat_id(lookup_location_id), lookup_country_location_id = normalize_heat_id(lookup_country_location_id) ) %>% mutate( ucc_location_id = coalesce(ucc_location_id, lookup_location_id), ucc_country_location_id = coalesce(ucc_country_location_id, lookup_country_location_id) ) %>% select(-any_of(c("lookup_location_id", "lookup_country_location_id"))) missing_loc_ids <- scenarios %>% filter(is.na(ucc_location_id) | is.na(ucc_country_location_id)) if (nrow(missing_loc_ids) > 0) { warning( "Some scenarios are missing location IDs after lookup (these may fail): ", paste(missing_loc_ids[[scenario_label_column]], collapse = ", ") ) } } # --------------------------------------------------------- # # Step 6. Check which scenario columns match template ---- # --------------------------------------------------------- # # The loop will only update columns that exist in the HEAT template. # Extra columns (e.g., labels, helper columns) are ignored. template_fields <- names(template_webapp_input) scenario_fields <- names(scenarios) matching_fields <- intersect(scenario_fields, template_fields) non_matching_fields <- setdiff(scenario_fields, template_fields) message(length(matching_fields), " scenario columns match fields in the template.") if (length(non_matching_fields) > 0) { message( "Ignoring non-template columns (this is often expected): ", paste(non_matching_fields, collapse = ", ") ) } if (length(matching_fields) == 0) { stop("No scenario columns match fields in the HEAT template. Check your input column names.") } # ---------------------------------------------------- # # Step 7. Helper functions for API call + updates ---- # ---------------------------------------------------- # apply_scenario_to_template <- function(template_input, scenario_row) { updated_input <- template_input for (field_name in names(scenario_row)) { if (field_name %in% names(updated_input)) { updated_input[[field_name]] <- scenario_row[[field_name]] } } updated_input } call_heat_api <- function(webapp_input, cookie, api_url) { # IMPORTANT: Small changes to JSON shape (e.g. auto_unbox) # or multipart field structure can trigger server-side errors. response <- httr::POST( api_url, httr::set_cookies(user = cookie), body = jsonlite::toJSON(list(webapp_input = webapp_input)), encode = "multipart" ) response_text <- httr::content(response, as = "text", encoding = "UTF-8") if (httr::http_error(response)) { return(list( ok = FALSE, status_code = httr::status_code(response), error_message = paste("HTTP error", httr::status_code(response), "-", response_text), payload = NULL )) } payload <- tryCatch( jsonlite::fromJSON(response_text), error = function(e) NULL ) if (is.null(payload)) { return(list( ok = FALSE, status_code = httr::status_code(response), error_message = "Could not parse JSON response from HEAT API.", payload = NULL )) } if ("error" %in% names(payload)) { return(list( ok = FALSE, status_code = httr::status_code(response), error_message = as.character(payload$error), payload = payload )) } list( ok = TRUE, status_code = httr::status_code(response), error_message = NULL, payload = payload ) } # --------------------------------------- # # Step 8. Run the assessment loop ---- # --------------------------------------- # results_list <- vector("list", length = nrow(scenarios)) results_raw_list <- vector("list", length = nrow(scenarios)) error_log <- vector("list", length = 0) for (i in seq_len(nrow(scenarios))) { scenario_row <- scenarios[i, , drop = FALSE] scenario_label <- as.character(scenario_row[[scenario_label_column]][1]) message(sprintf("[%s/%s] Running scenario: %s", i, nrow(scenarios), scenario_label)) current_input <- apply_scenario_to_template(template_webapp_input, scenario_row) api_result <- call_heat_api(current_input, cookie_value, heat_api_url) if (!isTRUE(api_result$ok)) { error_log[[length(error_log) + 1]] <- tibble::tibble( run_index = i, scenario = scenario_label, status_code = api_result$status_code, error_message = api_result$error_message ) message(" -> Failed: ", api_result$error_message) if (i < nrow(scenarios)) { Sys.sleep(wait_seconds_between_calls) } next } scenario_results <- api_result$payload$results scenario_results_raw <- api_result$payload$results_raw if (is.null(scenario_results) || NROW(scenario_results) == 0) { error_log[[length(error_log) + 1]] <- tibble::tibble( run_index = i, scenario = scenario_label, status_code = api_result$status_code, error_message = "No `results` returned by API." ) message(" -> Failed: No results returned.") if (i < nrow(scenarios)) { Sys.sleep(wait_seconds_between_calls) } next } scenario_results <- tibble::as_tibble(scenario_results) %>% mutate( run_index = i, scenario = scenario_label, .before = 1 ) if (!is.null(scenario_results_raw) && NROW(scenario_results_raw) > 0) { scenario_results_raw <- tibble::as_tibble(scenario_results_raw) %>% mutate( run_index = i, scenario = scenario_label, .before = 1 ) } else { scenario_results_raw <- tibble::tibble() message(" -> Warning: no `results_raw` returned by API for this scenario.") } # Optional: keep only selected pathways (e.g., Physical activity) if (!is.null(keep_pathways) && "pathway_label" %in% names(scenario_results)) { scenario_results <- scenario_results %>% filter(pathway_label %in% keep_pathways) } if (!is.null(keep_pathways) && "pathway_label" %in% names(scenario_results_raw)) { scenario_results_raw <- scenario_results_raw %>% filter(pathway_label %in% keep_pathways) } # Optional: remove rows with denominator == "NA" (string) if present if ("denominator" %in% names(scenario_results)) { scenario_results <- scenario_results %>% filter(is.na(denominator) | denominator != "NA") } if ("denominator" %in% names(scenario_results_raw)) { scenario_results_raw <- scenario_results_raw %>% filter(is.na(denominator) | denominator != "NA") } results_list[[i]] <- scenario_results results_raw_list[[i]] <- scenario_results_raw message(" -> Success") if (i < nrow(scenarios)) { Sys.sleep(wait_seconds_between_calls) } } # --------------------------------------- # # Step 9. Combine and save outputs ---- # --------------------------------------- # all_results <- dplyr::bind_rows(results_list) all_results_raw <- dplyr::bind_rows(results_raw_list) all_errors <- dplyr::bind_rows(error_log) if (nrow(all_results) == 0) { warning("No successful results were returned. Check `all_errors` and your inputs.") } # Save a workbook with multiple sheets (easy for sharing/review) openxlsx::write.xlsx( x = list( Results = all_results, ResultsRaw = all_results_raw, Errors = all_errors, Scenarios = scenarios ), file = output_xlsx_path, overwrite = TRUE ) # Save raw objects for re-use in R saveRDS( list( results = all_results, results_raw = all_results_raw, errors = all_errors, scenarios = scenarios, template_fields = sort(names(template_webapp_input)) ), file = output_rds_path ) message("Finished.") message("Results saved to: ", output_xlsx_path) message("R object saved to: ", output_rds_path) # ----------------------------------------------------------------- # # Checklist for adapting this script to your project ---- # ----------------------------------------------------------------- # # 1) Run one suitable HEAT assessment in the web tool and export the `.rds`. # 2) Put your batch input rows in a CSV (one row = one API run). # 3) Use the same HEAT field names as in the exported template wherever possible. # 4) If locations vary, join valid HEAT `location_id` and `country_location_id`. # 5) Test on 1-2 rows first. # 6) Run the full loop and spot-check outputs.