TTR5IFSG25VNBQ2F2FNOLUMTEIHVBHFOEXYB2ZWEHWOURUV4GJMQC
XI5ALEH6NPTQWB6O62QV62EP4H3K7WSNTHCOGT3LZIIU6I2YDGIQC
GQVS55HIQLU7KPJNRMF57QUM4EATSWFQRCS7ZEJMJPUXFX2NHSYAC
4QOTH75I2VINLY52J75OR3E3B5CN5ITHJGOAKQYXI6PH3XLJ6DBQC
2SABVMY3A2RZDF3KJXZSMJ2UQ4Q5EW422G4DVBJRKK26S2ESGVQAC
U4VCAFXQNTKC7KWWE3B2JJMZMFRGLBOHSZIOXCE6EEXW7WE2Q5NAC
GUXZCEWWPBCHXO26JVWZ74CTSDFDDO775YR7FKY7UGVZCA7GCSYAC
LVMGQJGH34VSGZCB74IQEJIVFNJE6OPQQCF7IPS72NPVMIUFBV4AC
5POF332LJEBGWEJUGI34P33Q4BQP7CAQNV5ODQT3PWG7FI7VNLOQC
77SIQZ3EGGV6KSECMLPDKQFGEC7CCFAPWGER7ZARQ5STDKJNU6GQC
4MG5JFXTKAE3SOVKGGNKEUTNCKOWEBHTGKVZHJWLWE3PTZTQKHPAC
CUADTSHQNPGMWHIJCWXKHNNIY4UJOYZ7XBZA5CJ2VVJJAMSNT4BAC
AT753JPOOWZCIYNKAG27LZSZZ72ZILWVENG42Y6U2S34JD3B6ZZQC
// Do we want meta for each line?
/// Create an empty `UIJson`.
pub fn new() -> Self {
UIJson(HashMap::new())
}
/// Insert a `UIGraphicJson` into `UIJson`. If the country already exists, increment the index
/// in the key until a space is available.
pub fn insert(&mut self, ui_graphic_json: UIGraphicJson) {
let mut ix = 0;
while self.0.get(&(ui_graphic_json.country, 0)).is_none() {
ix += 1;
}
self.0.insert((ui_graphic_json.country, ix), ui_graphic_json);
}
}
/// A UI plot. `UIGraphicJson` can be serialized to JSON.
pub struct UIGraphicJson {
country: Country,
title: String,
lines: Vec<UILineJson>,
}
/// ui_graphic:
/// time_series:
/// data_type: u
/// series: AUSURAMS_a
/// transform: ident
/// time_series:
/// data_type: i
/// series: _
/// transform: ident
/// country: Australia
/// index: 0
///
/// time_series:
/// data_type: u
/// series: AUSURAMS_a
/// time_series:
/// data_type: i
/// series: _
/// line:
impl UISpec {
// Convert UISpec into UIJson.
pub (crate) fn into_json(
&self,
data_spec: &IndexedCheckedDataSpec,
root_path: &str) -> Result<UIJson, Error>
{
let mut ui_json = UIJson::new();
for ui_graphic_spec in &self.0 {
let ui_graphic_json = ui_graphic_spec.into_json(
data_spec,
root_path,
)?;
ui_json.insert(ui_graphic_json);
}
Ok(ui_json)
}
}
impl UIGraphicSpec {
pub (crate) fn into_json(
&self,
data_spec: &IndexedCheckedDataSpec,
root_path: &str) -> Result<UIGraphicJson, Error>
{
let mut v = Vec::new();
for line_spec in &self.line {
let x_time_series_spec = &self.time_series[line_spec.x];
let y_time_series_spec = &self.time_series[line_spec.y];
let line_json = line_spec.into_json(
data_spec,
x_time_series_spec,
y_time_series_spec,
root_path,
).unwrap();
v.push(line_json);
}
Ok(
UIGraphicJson {
country: self.country,
title: self.title.clone(),
lines: v,
}
)
}
}
end_date: Option<MonthlyDate>,
end_date: Option<MonthlyDate>,
transform2: Transform2,
}
impl UILineSpec {
pub (crate) fn into_json(
&self,
data: &IndexedCheckedDataSpec,
x: &UITimeSeriesSpec,
y: &UITimeSeriesSpec,
root_path: &str) -> Result<UILineJson, Error>
{
x.assert_data_type(DataType::U)?;
y.assert_data_type(DataType::Inf)?;
let x_data = x.time_series_data(data, root_path)?;
let y_data = y.time_series_data(data, root_path)?;
let range = MonthlyDate::range(&self.start_date, &self.end_date);
let xy_data = match self.transform2 {
Transform2::Zip => {
let mut xy_data = x_data.zip_one_one(y_data);
xy_data.with_range(&range);
xy_data
}
};
Ok(
UILineJson {
data: xy_data,
}
)
}
x: self.value("line::x")?,
y: self.value("line::y")?,
start_date: self.opt_value("line::start_date")?,
end_date: self.opt_value("line::end_date")?,
x: self.value("line::x")?,
y: self.value("line::y")?,
start_date: self.opt_value("line::start_date")?,
end_date: self.opt_value("line::end_date")?,
transform2: self.value("line::transform2")?,
"ident" => Ok(Transform::Ident),
_ => Err(parse_transform(s)),
"ident" => Ok(Transform1::Ident),
_ => Err(parse_transform1(s)),
}
}
}
/// Transforms from two TimeSeries<1> to a TimeSeries<2>.
pub enum Transform2 {
///
Zip,
}
impl FromStr for Transform2 {
type Err = Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"zip" => Ok(Transform2::Zip),
_ => Err(parse_transform2(s)),
series: String,
transform: Transform,
series_id: SeriesId,
transform1: Transform1,
}
impl UITimeSeriesSpec {
pub (crate) fn assert_data_type(&self, data_type: DataType) -> Result<(), Error> {
if self.data_type == data_type {
Ok(())
} else {
Err(data_type_mismatch(&self.data_type.to_string(), &data_type.to_string()))
}
}
pub (crate) fn time_series_data(
&self,
data: &IndexedCheckedDataSpec,
root_path: &str) -> Result<RegularTimeSeries<1>, Error>
{
data.time_series_data(
self.series_id.clone(),
root_path
)
}
start_date: self.opt_value("time_series::start_date")?,
end_date: self.opt_value("time_series::end_date")?,
series: self.value("time_series::series")?,
transform: self.value("time_series::transform")?,
series_id: self.value("time_series::series")?,
transform1: self.value("time_series::transform1")?,
/// country:
/// name: Australia
/// graphic:
/// series:
/// id: AUSURANAA
/// id: AUSURAQS
/// id: AUSURHARMADSMEI
/// id: AUSURHARMMDSMEI
/// country: Australia
/// index: 0
/// graphic:
/// series:
/// series_id: AUSURANAA
/// data_type: u
/// series:
/// data_type: u
/// series_id: AUSURAQS
/// series:
/// data_type: u
/// series_id: AUSURHARMADSMEI
/// series:
/// data_type: u
/// series_id: AUSURHARMMDSMEI
///
/// ```
/// graphic:
/// series:
/// series_id: AUSURANAA
/// data_type: u
/// series:
/// data_type: u
/// series_id: AUSURAQS
/// series:
/// data_type: u
/// series_id: AUSURHARMADSMEI
/// series:
/// data_type: u
/// series_id: AUSURHARMMDSMEI
/// ```
series_ids: self.vec_value("graphic::data")?,
series: self.vec_at("graphic::series")?,
}
)
}
}
/// Component of `TSGraphicSpec`.
/// ```text
/// series:
/// data_type: u
/// series_id: AUSURHARMADSMEI
/// ```
pub struct TSSeriesSpec {
///
data_type: DataType,
series_id: SeriesId,
}
impl TSSeriesSpec {
/// Return a new `TSSeries`.
pub fn new(data_type: DataType, series_id: SeriesId) -> Self {
TSSeriesSpec {
data_type,
series_id,
}
}
}
impl<'a> TryInto<TSSeriesSpec> for KeyTreeRef<'a> {
type Error = keytree::Error;
fn try_into(self) -> Result<TSSeriesSpec, Self::Error> {
Ok(
TSSeriesSpec {
data_type: self.value("series::data_type")?,
series_id: self.value("series::series_id")?,
// // "usa;interest",
// // "prime",
// // "australia;interest" -> nothing
// for item in Fred::tags_series("australia;interest")
// .unwrap()
// .series()
// .iter()
// {
// println!("{}", item);
// // println!("{}", item.tags());
// }
// let spec_path = shellexpand::tilde("~/currency.engineering/source_spec.keytree");
// let data_path = shellexpand::tilde("~/test_data");
// Read in the data specification.
let data_spec = CheckedDataSpec::from_file("checked_data.keytree").into_indexed();
let ts_spec = TSSpec::from_file("ts_spec.keytree");
// spec_to_data::save_data(&spec_path, &data_path);
// println!("{}", build_data_spec().keytree());
let spec = DataSpec::from_file("source_data.keytree");
let mut checked = spec.check();
checked.write("/home/dyppb/test_data");
// let ts_data = SourceData::new();
checked.bootstrap_ts_spec();
let ts_json = ts_spec.into_json(&data_spec, &root_dir);
//! Collect unemployment rate, inflation rate and interest rate data server side.
//! Collect unemployment rate, inflation rate and interest rate data server side, then serve this
//! data as JSON in a format suitable for building UI graphics using D3.
//! Check each data series, to see if the data is well-formed. Save the output specification.
//! Check each data series, to see if the data is well-formed.
//! ```
//! let spec = DataSpec::from_file("source_data.keytree");
//! println!("{}", spec.check().keytree());
//! ```
//! Save the output specification in `checked_data.keytree`.
//! checked_data_spec
//! .from_file("data_spec.keytree")
//! .bootstrap_ts_spec;
//! ```
//! The specification will look something like
//! let checked = CheckedDataSpec::from_file("checked_data.keytree");
//! checked.write("/full/path/to/data");
//! ```
//!
//! If there is a break in the connection we can use
//! page:
//! country:
//! name: Australia
//! graphic:
//! series:
//! id: AUSURANAA
//! id: AUSURAQS
//! id: AUSURHARMADSMEI
//! id: AUSURHARMMDSMEI
//! checked.resume_write(Series::new("GBRURNAA"), "/full/path/to/data")
//! ```
//! to resume.
//!
//! ## Step 5. Generate a generic time-series graphics specification.
//!
//! This is used to generate time-series plots. Once it is generated in full, then it can be edited
//! manually.
//!
//! ```text
//! checked.generic_ts_spec()
//! // Load all data to memory. The server uses `ts_json` as a data-store to respond to data
//! // requests.
//! let ts_json = ts_spec.into_json(data_spec);
//! ```
//! let ts_json = ts_spec.into_json(data_spec, "/full/path/to/data");
/// Get data_type and country of a given series_id.
pub (crate) fn get(&self, series_id: SeriesId) -> (DataType, Country) {
let index = self.index.get(&series_id).unwrap();
let data_type = self.data.0[*index].data_type;
let country = self.data.0[*index].country;
(data_type, country)
}
/// Return a generic time-series graphics specification.
///
/// ```text
/// ts_spec:
/// page:
/// country: Australia
/// index: 0
/// graphic:
/// series:
/// series_id: AUSURANAA
/// data_type: u
/// series:
/// data_type: u
/// series_id: AUSURAQS
/// series:
/// data_type: u
/// series_id: AUSURHARMADSMEI
/// series:
/// data_type: u
/// series_id: AUSURHARMMDSMEI
/// ```
/// TSSpec
/// TSPageSpec
/// country
/// index
/// TSGraphicSpec
/// height
/// TSSeriesSpec
///
/// IndexedCheckedDataSpec
/// CheckedDataSpec
/// CheckedSourceSeries
/// DataType
/// Country
/// SeriesId
///
pub fn generic_ts_spec(&self) -> TSSpec {
let mut map: BTreeMap<(DataType, Country), Vec<SeriesId>> = BTreeMap::new();
for (series_id, _) in self.index.iter() {
let (data_type, country) = self.get(series_id.clone());
match map.get_mut(&(data_type, country)) {
Some(value) => value.push(series_id.clone()),
None => {
map.insert((data_type, country), vec!(series_id.clone()));
},
};
}
// build TSSpec
let ts_spec = TSSpec::new();
for ((data_type, country), series_ids) in map {
let mut ts_spec = TSSpec::new();
let mut ts_graphic_spec = TSGraphicSpec::new();
for series_id in series_ids {
let ts_series_spec = TSSeriesSpec::new(data_type, series_id);
ts_graphic_spec.push(ts_series_spec);
};
pub fn write(&mut self, root_path: &str) {
pub fn write(&self, root_path: &str) {
for series in &self.0 {
series.write_data(root_path);
series.write_meta(root_path);
}
}
/// Same as `write()` except that it starts in the specification at `series_id`. Useful if there
/// is a break in the connection when writing.
pub fn resume_write(&self, series_id: SeriesId, root_path: &str) {
/// Takes `checked_data.keytree` and builds a collection of graphics,
/// each plotting 1 time-series.
pub fn bootstrap_ts_spec(&self) {
let mut h: HashMap<PageKey, TSPageSpec> = HashMap::new();
for series in &self.0 {
// Check that there are no errors in the CheckedDataSpec series.
if series.error.to_lowercase() != "ok" {
println!("{}", series.keytree());
panic!();
};
// Insert the series into the TSGraphicSpec.
match h.get_mut(&series.key()) {
Some(page_spec) => {
page_spec.graphics[0].push(&series.id);
},
None => {
let mut ts_graphic_spec = TSGraphicSpec::new();
ts_graphic_spec.push(&series.id);
let ts_page_spec = TSPageSpec::new(series.key(), vec!(ts_graphic_spec));
h.insert(series.key(), ts_page_spec);
}
}
}
// Build TSSpec from the hashmap.
let mut ts_spec = TSSpec(Vec::new());
for (key, page_spec) in h {
ts_spec.push(page_spec);
};
println!("{}", ts_spec.keytree())
}
// self.time_stamp = Some(TimeStamp::now());
}
/// Return relevant inflation rate series for a country. This function first selects all series with
/// a certain tag pattern, and then applies required phrases and exclusionary phrases.
/// ```
/// println!("{}", generic_inf_series_spec(Country::Canada));
/// ```
pub fn generic_inf_series_spec(country: Country) -> SeriesItems {
if let Country::UnitedStates = country {
// Need to use a different search technique for US data.
let exclude_phrase = vec!(
"Producer Price Index",
"Projections",
"Export Price Index",
"Implicit Price Deflator",
"Employment Cost Index",
"Contributions to",
"Inflation Expectation",
"Equity Market",
"excluding food and energy",
"Excluding Food and Energy",
"excluding Food and Energy",
"Urban",
"Sticky",
"Opinion",
"Consumer Price Index",
"Personal Consumption",
"Private Consumption",
);
// let one_of = vec!(
// "Unemployment Rate for United States",
// "Unemployment Rate: Aged 15 and Over: All Persons for the United States",
// "Unemployment Rate: Aged 15-74: All Persons for the United States",
// "Harmonized Unemployment Rate: Total: All Persons for the United States",
// "Unemployment Rate - 18 Years and Over",
// );
let tag_series = Fred::tags_series("inflation;usa;nation").unwrap().seriess;
tag_series
.exclude_phrases(exclude_phrase)
// .equals_one_of(one_of)
} else {
let (exclude_phrase, include_phrase) = match country {
Country::Australia => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Austria => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Belgium => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Canada => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Chile => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::CzechRepublic => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Denmark => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Estonia => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Finland => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::France => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Germany => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Greece => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Ireland => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Israel => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Italy => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Japan => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Latvia => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Netherlands => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::NewZealand => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Norway => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Poland => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Serbia => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::SouthKorea => {
(
vec!(
),
vec!(
"Inflation",
),
)
},
Country::Spain => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Sweden => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::Switzerland => {
(
vec!(
"Opinion",
),
vec!(
"Inflation",
),
)
},
Country::UnitedKingdom => {
(
vec!(
"Opinion",
"Consumer Price Inflation",
),
vec!(
"Inflation",
),
)
},
Country::UnitedStates => {
(
vec!(
),
vec!(
"",
),
)
},
_ => {
panic!()
},
};
match Fred::tags_series(&to_tag("inflation", country)) {
Ok(tags_series) => {
tags_series.seriess
.exclude_phrases(exclude_phrase)
.only_include(include_phrase)
},
Err(err) => {
println!("{}", err);
panic!();
},
}
}
/// Build a generic specification of one datatype and country.
pub fn data_spec_for_country(data_type: DataType, country: Country) -> DataSpec {
let mut seriess = DataSpec(Vec::new());
match data_type {
DataType::U => {
for series_item in generic_u_series_spec(country).iter() {
seriess.0.push(
SourceSeries {
data_type: DataType::U,
country,
series_id: SeriesId::new(&series_item.id.clone()),
}
)
}
println!("{} {}", country, data_type);
},
DataType::Inf => {
for series_item in generic_inf_series_spec(country).iter() {
seriess.0.push(
SourceSeries {
data_type: DataType::Inf,
country,
series_id: SeriesId::new(&series_item.id.clone()),
}
)
}
println!("{} {}", country, data_type);
},
DataType::Cpi => {
for series_item in generic_cpi_series_spec(country).iter() {
seriess.0.push(
SourceSeries {
data_type: DataType::Cpi,
country,
series_id: SeriesId::new(&series_item.id.clone()),
}
)
}
println!("{} {}", country, data_type);
},
DataType::Int => {},
}
seriess
}
/// Build a generic specification for all data. Output looks like
println!("{}", country);
for series_item in generic_u_series_spec(&country).iter() {
seriess.0.push(
SourceSeries {
data_type,
country,
id: series_item.id.clone(),
}
)
}
}
seriess.append(&mut data_spec_for_country(DataType::U, country));
}
for country in countries_with_data() {
seriess.append(&mut data_spec_for_country(DataType::Cpi, country));
}
for country in countries_with_data() {
seriess.append(&mut data_spec_for_country(DataType::Inf, country));
}
// /// Read in source_data.keytree, download series, and return
// /// `CheckedSourceSeries`.
// pub fn update(&self) {
// for series in &self.0 {
// let data = match Fred::series(&series.id) {
// Ok(data) => println!("{}", data.keytree()),
// Err(err) => println!("{:?}", err),
// };
// }
// }
/// Append `other` to `Self`.
pub fn append(&mut self, other: &mut DataSpec) {
self.0.append(&mut other.0)
}