boogerlad/mj - Change XMKLB6DI3SLWQ6DKAAL47VNZQGER5NRJKCPLMVZO4PYHGM3NQJAQC

the store table has addresses parsed by google. for the unique constraint to work on insert, parsed addresses are required. the store scraper naively attempts to insert every store. this works, but is expensive(~10usd). introduce store_temp table which contains the raw addresses from the previous scrape to insert only new stores and show removed stores. should probably truncate store_temp once a month to update longitude, latitude and open times

Created by boogerlad on August 24, 2020

XMKLB6DI3SLWQ6DKAAL47VNZQGER5NRJKCPLMVZO4PYHGM3NQJAQC

Dependencies

In channels

main

Change contents

Insertion in initialize.sql at line 48 [4.17]

[2.209]

[3.146]

create table store_temp (
	joined_name citext,
	address citext,
	address2 citext,
	city citext,
	region citext,
	country citext,
	postal_code citext
);

Insertion in scrape/common.js at line 104 [5.4897]

[2.5072]

			location.lng = (new Big(location.lng)).toFixed(7);
			location.lat = (new Big(location.lat)).toFixed(7);

Insertion in scrape/stores.js at line 13 [6.2622]
[6.7664]
[6.7664]
```
	let idx = [];
```
Deletion in scrape/stores.js at line 16 [6.2622]
B:BD[6.7702] → [6.7702:7723]
```
	let address2 = [];
```

Insertion in scrape/stores.js at line 17 [6.2622]

[6.7740]

	let postal_code = [];
	for(let i = 0; i < ab.length; ++i) {
		ab[i] = ab[i].split('\t');
		idx.push(i);
		names.push(ab[i][1] = sanitize_name(ab[i][1]));
		address.push(ab[i][2]);
		postal_code.push(ab[i][3]);
		city.push(ab[i][0]);
	}
	await postgres.query(
		`create temporary table alberta as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[],
			$5::citext[]
		) as t(idx, joined_name, address, city, postal_code)`,
		[idx, names, address, city, postal_code]
	);
	idx = (await postgres.query(`
		select idx
		from alberta
		where not exists (
			select
			from store_temp
			where
				joined_name = alberta.joined_name and
				address = alberta.address and
				address2 = '' and
				city = alberta.city and
				region = 'alberta' and
				country = 'canada' and
				postal_code = alberta.postal_code
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];
	let address2 = [];
	city = [];

Replacement in scrape/stores.js at line 60 [6.2622]
B:BD[6.7779] → [6.7779:7803]
```
	let postal_code = [];
```
[6.7779]
[6.7803]
```
	postal_code = [];
```

Replacement in scrape/stores.js at line 67 [6.2622]

B:BD[6.7938] → [6.7938:8007]

	for(let i = 0; i < ab.length; ++i) {
		let t = ab[i].split('\t');

[6.7938]

[6.8007]

	for(let i = 0; i < idx.length; ++i) {
		let t = ab[idx[i]];

Replacement in scrape/stores.js at line 74 [6.2622]
B:BD[6.8123] → [6.8123:8146]
```
			console.log(t[1]);
```
[6.8123]
[6.8146]
```
			console.log(t);
```
Replacement in scrape/stores.js at line 86 [6.2622]
B:BD[6.8416] → [6.8416:8452]
```
		names.push(sanitize_name(t[1]));
```
[6.8416]
[6.8452]
```
		names.push(t[1]);
```
Insertion in scrape/stores.js at line 206 [6.2622]
[6.11748]
[6.11748]
```
		let idx = [];
		let jdx = [];
```
Deletion in scrape/stores.js at line 210 [6.2622]
B:BD[6.11788] → [6.11788:11810]
```
		let address2 = [];
```

Insertion in scrape/stores.js at line 211 [6.2622]

[6.11828]

		for(let i = 0; i < idk.length; ++i) {
			for(let j = 0; j < results[i].length; ++j) {
				idx.push(i);
				jdx.push(j);
				names.push(results[i][j][0] = sanitize_name(results[i][j][0]));
				address.push(results[i][j][1]);
				city.push(idk[i]);
			}
		}
		await postgres.query(
			`create temporary table manitoba as 
			select *
			from unnest (
				$1::smallint[],
				$2::smallint[],
				$3::citext[],
				$4::citext[],
				$5::citext[]
			) as t(idx, jdx, joined_name, address, city)`,
			[idx, jdx, names, address, city]
		);
		let temp = (await postgres.query(`
			select idx, jdx
			from manitoba
			where not exists (
				select
				from store_temp
				where
					joined_name = manitoba.joined_name and
					address = manitoba.address and
					address2 = '' and
					city = manitoba.city and
					region = 'manitoba' and
					country = 'canada' and
					postal_code = ''
			)
		`)).rows;
		idx = temp.map(x => x.idx);
		jdx = temp.map(x => x.jdx);
		names = [];
		address = [];
		let address2 = [];
		city = [];

Replacement in scrape/stores.js at line 262 [6.2622]

B:BD[6.12016] → [6.12016:13061]

		for(let i = 0; i < idk.length; ++i) {
			for(let j = 0; j < results[i].length; ++j) {
				let lol = sanitize_name(results[i][j][0]);
				if(results[i][j][1] === 'Otineka Mall' && lol === 'Meta') {
					names.push(lol);
					address.push('Otineka Mall Highway 10 North');//gmaps ok
					address2.push('Unit 128');
					city.push('Opaskwayak');
					region.push('Manitoba');
					country.push('Canada');
					postal_code.push('R0B2J0');
					longitude.push('-101.261585');
					latitude.push('53.834096');
					osrm_hint_foot.push(await request(`http://127.0.0.1:5000/nearest/v1/fuck/-101.261585,53.834096`).then(x => JSON.parse(x).waypoints[0].hint));
					osrm_hint_car.push(await request(`http://127.0.0.1:5001/nearest/v1/fuck/-101.261585,53.834096`).then(x => JSON.parse(x).waypoints[0].hint));
					timezone.push('America/Winnipeg');
				} else {
					let r;
					try {
						r = await g(`${results[i][j][1]} ${idk[i]} manitoba canada`);
					} catch(e) {
						console.error(e);
						console.log(results[i][j][0]);

[6.12016]

[6.13061]

		for(let k = 0; k < idx.length; ++k) {
			const i = idx[k];
			const j = jdx[k];
			if(results[i][j][1] === 'Otineka Mall' && results[i][j][0] === 'Meta') {
				names.push(results[i][j][0]);
				address.push('Otineka Mall Highway 10 North');//gmaps ok
				address2.push('Unit 128');
				city.push('Opaskwayak');
				region.push('Manitoba');
				country.push('Canada');
				postal_code.push('R0B2J0');
				longitude.push('-101.261585');
				latitude.push('53.834096');
				osrm_hint_foot.push(await request(`http://127.0.0.1:5000/nearest/v1/fuck/-101.261585,53.834096`).then(x => JSON.parse(x).waypoints[0].hint));
				osrm_hint_car.push(await request(`http://127.0.0.1:5001/nearest/v1/fuck/-101.261585,53.834096`).then(x => JSON.parse(x).waypoints[0].hint));
				timezone.push('America/Winnipeg');
			} else {
				let r;
				try {
					r = await g(`${results[i][j][1]} ${idk[i]} manitoba canada`);
				} catch(e) {
					console.error(e);
					console.log(results[i][j]);
					continue;
				}
				if(!pc.test(r[0].postal_code)) {
					if(results[i][j][1] === '300A North Railway Street') {
						r[0].postal_code = 'R6M1S7';
					} else {
						console.error('postal_code goofup', r, results[i][j], idk[i]);

Deletion in scrape/stores.js at line 293 [6.2622]

B:BD[6.13078] → [6.13078:13338]

					}
					if(!pc.test(r[0].postal_code)) {
						if(results[i][j][1] === '300A North Railway Street') {
							r[0].postal_code = 'R6M1S7';
						} else {
							console.error('postal_code goofup', r, results[i][j], idk[i]);
							continue;
						}

Deletion in scrape/stores.js at line 294 [6.2622]

B:BD[6.13346] → [6.13346:13787]

					names.push(lol);
					address.push(r[0].address);
					address2.push(results[i][j][1] === '300A North Railway Street' ? 'A' : r[0].address2);
					city.push(r[0].city);
					region.push(r[0].region);
					country.push(r[0].country);
					postal_code.push(r[0].postal_code);
					longitude.push(r[1].lng);
					latitude.push(r[1].lat);
					osrm_hint_foot.push(r[3]);
					osrm_hint_car.push(r[4]);
					timezone.push(r[2]);

Insertion in scrape/stores.js at line 295 [6.2622]

[6.13794]

				names.push(results[i][j][0]);
				address.push(r[0].address);
				address2.push(results[i][j][1] === '300A North Railway Street' ? 'A' : r[0].address2);
				city.push(r[0].city);
				region.push(r[0].region);
				country.push(r[0].country);
				postal_code.push(r[0].postal_code);
				longitude.push(r[1].lng);
				latitude.push(r[1].lat);
				osrm_hint_foot.push(r[3]);
				osrm_hint_car.push(r[4]);
				timezone.push(r[2]);

Insertion in scrape/stores.js at line 406 [6.2622]
[6.16631]
[6.16631]
```
	let idx = [];
```
Deletion in scrape/stores.js at line 409 [6.2622]
B:BD[6.16669] → [6.16669:16690]
```
	let address2 = [];
```

Insertion in scrape/stores.js at line 410 [6.2622]

[6.16707]

	for(let i = 0; i < results.length; ++i) {
		idx.push(i);
		names.push(results[i][0] = sanitize_name(results[i][0]));
		address.push(results[i][1]);
		city.push(results[i][2]);
	}
	await postgres.query(
		`create temporary table saskatchewan as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[]
		) as t(idx, joined_name, address, city)`,
		[idx, names, address, city]
	);
	idx = (await postgres.query(`
		select idx
		from saskatchewan
		where not exists (
			select
			from store_temp
			where
				joined_name = saskatchewan.joined_name and
				address = saskatchewan.address and
				address2 = '' and
				city = saskatchewan.city and
				region = 'saskatchewan' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];
	let address2 = [];
	city = [];

Replacement in scrape/stores.js at line 456 [6.2622]

B:BD[6.16903] → [6.16903:16947]

	for(let i = 0; i < results.length; ++i) {

[6.16903]

[6.16947]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Replacement in scrape/stores.js at line 472 [6.2622]
B:BD[6.17763] → [6.17763:17818]
```
		} else if(results[i][0] === 'Vatic Cannabis Co.') {
```
[6.17763]
[6.17818]
```
		} else if(results[i][0] === 'Vatic') {
```
Replacement in scrape/stores.js at line 496 [6.2622]
B:BD[6.18815] → [6.18815:18848]
```
				console.log(results[i][0]);
```
[6.18815]
[6.18848]
```
				console.log(results[i]);
```
Replacement in scrape/stores.js at line 503 [6.2622]
B:BD[6.18984] → [6.18984:19030]
```
			names.push(sanitize_name(results[i][0]));
```
[6.18984]
[6.19030]
```
			names.push(results[i][0]);
```
Insertion in scrape/stores.js at line 645 [6.2622]
[6.22892]
[6.22892]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 647 [6.2622]

[6.22912]

	for(let i = 0; i < results.length; ++i) {
		idx.push(i);
		address.push(results[i][1]);
	}
	await postgres.query(
		`create temporary table newbrunswick as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[]
		) as t(idx, address)`,
		[idx, address]
	);
	idx = (await postgres.query(`
		select idx
		from newbrunswick
		where not exists (
			select
			from store_temp
			where
				joined_name = '' and
				address = newbrunswick.address and
				address2 = '' and
				city = '' and
				region = 'newbrunswick' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);
	address = [];

Replacement in scrape/stores.js at line 688 [6.2622]

B:BD[6.23147] → [6.23147:23191]

	for(let i = 0; i < results.length; ++i) {

[6.23147]

[6.23191]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Replacement in scrape/stores.js at line 718 [6.2622]
B:BD[6.24654] → [6.24654:24687]
```
				console.log(results[i][0]);
```
[6.24654]
[6.24687]
```
				console.log(results[i][1]);
```

Replacement in scrape/stores.js at line 722 [6.2622]

B:BD[6.24745] → [6.24745:24802]

				console.error('postal_code goofup', r, results[i]);

[6.24745]

[6.24802]

				console.error('postal_code goofup', r, results[i][1]);

Insertion in scrape/stores.js at line 842 [6.2622]
[6.28268]
[6.28268]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 845 [6.2622]

[6.28306]

	let city = [];
	let postal_code = [];
	for(let i = 0; i < bc.length; ++i) {
		if(bc[i].Status === 'Open') {
			idx.push(i);
			names.push(bc[i].Name = sanitize_name(bc[i].Name));
			address.push(bc[i].Address);
			city.push(bc[i].City);
			postal_code.push(bc[i].Postal);
		}
	}
	await postgres.query(
		`create temporary table britishcolumbia as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[],
			$5::citext[]
		) as t(idx, joined_name, address, city, postal_code)`,
		[idx, names, address, city, postal_code]
	);
	idx = (await postgres.query(`
		select idx
		from britishcolumbia
		where not exists (
			select
			from store_temp
			where
				joined_name = britishcolumbia.joined_name and
				address = britishcolumbia.address and
				address2 = '' and
				city = britishcolumbia.city and
				region = 'britishcolumbia' and
				country = 'canada' and
				postal_code = britishcolumbia.postal_code
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];

Replacement in scrape/stores.js at line 887 [6.2622]
B:BD[6.28327] → [6.28327:28344]
```
	let city = [];
```
[6.28327]
[6.28344]
```
	city = [];
```
Replacement in scrape/stores.js at line 890 [6.2622]
B:BD[6.28383] → [6.28383:28407]
```
	let postal_code = [];
```
[6.28383]
[6.28407]
```
	postal_code = [];
```

Replacement in scrape/stores.js at line 897 [6.2622]

B:BD[6.28542] → [6.28542:28797]

	for(let i = 0; i < bc.length; ++i) {
		if(bc[i].Status === 'Open') {
			let r;
			try {
				r = await g(`${bc[i].Address} ${bc[i].City} british columbia ${bc[i].Postal} canada`);
			} catch(e) {
				console.error(e);
				console.log(bc[i].Name);

[6.28542]

[6.28797]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];
		let r;
		try {
			r = await g(`${bc[i].Address} ${bc[i].City} british columbia ${bc[i].Postal} canada`);
		} catch(e) {
			console.error(e);
			console.log(bc[i]);
			continue;
		}
		if(!pc.test(r[0].postal_code)) {
			bc[i].Postal = bc[i].Postal.replace(/\s+/g, '');
			if(r[0].postal_code.length === 3 && bc[i].Postal.length === 6) {
				r[0].postal_code = bc[i].Postal
			} else {
				console.error('postal_code goofup', r, bc[i]);

Deletion in scrape/stores.js at line 915 [6.2622]

B:BD[6.28818] → [6.28818:29524]

			if(!pc.test(r[0].postal_code)) {
				bc[i].Postal = bc[i].Postal.replace(/\s+/g, '');
				if(r[0].postal_code.length === 3 && bc[i].Postal.length === 6) {
					r[0].postal_code = bc[i].Postal
				} else {
					console.error('postal_code goofup', r, bc[i]);
					continue;
				}
			}
			names.push(sanitize_name(bc[i].Name));
			address.push(r[0].address);
			address2.push(r[0].address2);
			city.push(r[0].city);
			region.push(r[0].region);
			country.push(r[0].country);
			postal_code.push(r[0].postal_code);
			longitude.push(r[1].lng);
			latitude.push(r[1].lat);
			osrm_hint_foot.push(r[3]);
			osrm_hint_car.push(r[4]);
			timezone.push(r[2]);
			phone.push(bc[i].Phone);

Insertion in scrape/stores.js at line 916 [6.2622]

[6.29529]

		names.push(bc[i].Name);
		address.push(r[0].address);
		address2.push(r[0].address2);
		city.push(r[0].city);
		region.push(r[0].region);
		country.push(r[0].country);
		postal_code.push(r[0].postal_code);
		longitude.push(r[1].lng);
		latitude.push(r[1].lat);
		osrm_hint_foot.push(r[3]);
		osrm_hint_car.push(r[4]);
		timezone.push(r[2]);
		phone.push(bc[i].Phone);

Insertion in scrape/stores.js at line 1014 [6.2622]
[6.31878]
[6.31878]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 1019 [6.2622]

[6.31954]

	let postal_code = [];
	for(let i = 0; i < nl.length; ++i) {
		idx.push(i);
		names.push(nl[i].name = sanitize_name(nl[i].name));
		address.push(nl[i].address);
		address2.push(nl[i].address2);
		city.push(nl[i].city);
		postal_code.push(nl[i].postal_zip);
	}
	await postgres.query(
		`create temporary table newfoundland as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[],
			$5::citext[],
			$6::citext[]
		) as t(idx, joined_name, address, address2, city, postal_code)`,
		[idx, names, address, address2, city, postal_code]
	);
	idx = (await postgres.query(`
		select idx
		from newfoundland
		where not exists (
			select
			from store_temp
			where
				joined_name = newfoundland.joined_name and
				address = newfoundland.address and
				address2 = newfoundland.address2 and
				city = newfoundland.city and
				region = 'newfoundland' and
				country = 'canada' and
				postal_code = newfoundland.postal_code
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];
	address2 = [];
	city = [];

Replacement in scrape/stores.js at line 1063 [6.2622]
B:BD[6.31993] → [6.31993:32017]
```
	let postal_code = [];
```
[6.31993]
[6.32017]
```
	postal_code = [];
```
Replacement in scrape/stores.js at line 1071 [6.2622]
B:BD[6.32174] → [6.32174:32213]
```
	for(let i = 0; i < nl.length; ++i) {
```
[6.32174]
[6.32213]
```
	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];
```
Replacement in scrape/stores.js at line 1078 [6.2622]
B:BD[6.32379] → [6.32379:32408]
```
			console.log(nl[i].name);
```
[6.32379]
[6.32408]
```
			console.log(nl[i]);
```
Replacement in scrape/stores.js at line 1091 [6.2622]
B:BD[6.33152] → [6.33152:33194]
```
		names.push(sanitize_name(nl[i].name));
```
[6.33152]
[6.33194]
```
		names.push(nl[i].name);
```
Insertion in scrape/stores.js at line 1194 [6.2622]
[6.36003]
[6.36003]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 1198 [6.2622]

[6.36061]

	let postal_code = [];
	for(let i = 0; i < x.length; ++i) {
		for(let j = 0; j < x[i].features.length; ++j) {
			if(x[i].features[j].featureId === 'CP') {
				idx.push(i);
				address.push(x[i].addressInfo.address1);
				address2.push(x[i].addressInfo.address2 ?? '');
				city.push(x[i].addressInfo.city);
				postal_code.push(x[i].addressInfo.postal);
				break;
			}
		}
	}
	await postgres.query(
		`create temporary table novascotia as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[],
			$5::citext[]
		) as t(idx, address, address2, city, postal_code)`,
		[idx, address, address2, city, postal_code]
	);
	idx = (await postgres.query(`
		select idx
		from novascotia
		where not exists (
			select
			from store_temp
			where
				joined_name = '' and
				address = novascotia.address and
				address2 = novascotia.address2 and
				city = novascotia.city and
				region = 'novascotia' and
				country = 'canada' and
				postal_code = novascotia.postal_code
		)
	`)).rows.map(x => x.idx);
	address = [];
	address2 = [];
	city = [];

Replacement in scrape/stores.js at line 1244 [6.2622]
B:BD[6.36100] → [6.36100:36124]
```
	let postal_code = [];
```
[6.36100]
[6.36124]
```
	postal_code = [];
```

Replacement in scrape/stores.js at line 1252 [6.2622]

B:BD[6.36276] → [6.36276:36679]

	for(let i = 0; i < x.length; ++i) {
		for(let j = 0; j < x[i].features.length; ++j) {
			if(x[i].features[j].featureId === 'CP') {
				let r;
				try {
					r = await g(`${x[i].addressInfo.address1} ${x[i].addressInfo.address2 ?? ''} ${x[i].addressInfo.city} nova scotia ${x[i].addressInfo.postal} canada`);
				} catch(e) {
					console.error(e);
					console.log(x[i].name);
					continue;

[6.36276]

[6.36679]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];
		let r;
		try {
			r = await g(`${x[i].addressInfo.address1} ${x[i].addressInfo.address2 ?? ''} ${x[i].addressInfo.city} nova scotia ${x[i].addressInfo.postal} canada`);
		} catch(e) {
			console.error(e);
			console.log(x[i].name);
			continue;
		}
		if(!pc.test(r[0].postal_code)) {
			x[i].postal = x[i].postal.replace(/\s+/g, '');
			if(r[0].postal_code.length === 3 && x[i].postal.length === 6) {
				r[0].postal_code = x[i].postal
			} else {
				console.error('postal_code goofup', r, x[i]);
				continue;
			}
		}
		address.push(r[0].address);
		address2.push(r[0].address2);
		city.push(r[0].city);
		region.push(r[0].region);
		country.push(r[0].country);
		postal_code.push(r[0].postal_code);
		longitude.push(r[1].lng);
		latitude.push(r[1].lat);
		osrm_hint_foot.push(r[3]);
		osrm_hint_car.push(r[4]);
		timezone.push(r[2]);
		phone.push(x[i].phone);
		let tim = [];
		for(let k = 0; k < x[i].openingHours.length; ++k) {
			let d = 0;
			switch(x[i].openingHours[k].dayOfWeek) {
				case 'Tuesday': d = 1440; break;
				case 'Wednesday': d = 2880; break;
				case 'Thursday': d = 4320; break;
				case 'Friday': d = 5760; break;
				case 'Saturday': d = 7200; break;
				case 'Sunday': d = 8640; break; 
			}
			let beg = x[i].openingHours[k].opens.split(':').map(Number);
			beg = beg[0] * 60 + beg[1];
			let end = x[i].openingHours[k].closes.split(':').map(Number);
			end = end[0] * 60 + end[1];
			let rtn = [d + beg, d + end];
			if(end < beg) {
				if(d === 8640) {
					tim.push([rtn[0], 10080]);
					rtn[0] = 0;
					rtn[1] -= 8640;
				} else {
					rtn[1] += 1440;

Deletion in scrape/stores.js at line 1307 [6.2622]

B:BD[6.36686] → [6.36686:38253]

				if(!pc.test(r[0].postal_code)) {
					x[i].postal = x[i].postal.replace(/\s+/g, '');
					if(r[0].postal_code.length === 3 && x[i].postal.length === 6) {
						r[0].postal_code = x[i].postal
					} else {
						console.error('postal_code goofup', r, x[i]);
						continue;
					}
				}
				address.push(r[0].address);
				address2.push(r[0].address2);
				city.push(r[0].city);
				region.push(r[0].region);
				country.push(r[0].country);
				postal_code.push(r[0].postal_code);
				longitude.push(r[1].lng);
				latitude.push(r[1].lat);
				osrm_hint_foot.push(r[3]);
				osrm_hint_car.push(r[4]);
				timezone.push(r[2]);
				phone.push(x[i].phone);
				let tim = [];
				for(let k = 0; k < x[i].openingHours.length; ++k) {
					let d = 0;
					switch(x[i].openingHours[k].dayOfWeek) {
						case 'Tuesday': d = 1440; break;
						case 'Wednesday': d = 2880; break;
						case 'Thursday': d = 4320; break;
						case 'Friday': d = 5760; break;
						case 'Saturday': d = 7200; break;
						case 'Sunday': d = 8640; break; 
					}
					let beg = x[i].openingHours[k].opens.split(':').map(Number);
					beg = beg[0] * 60 + beg[1];
					let end = x[i].openingHours[k].closes.split(':').map(Number);
					end = end[0] * 60 + end[1];
					let rtn = [d + beg, d + end];
					if(end < beg) {
						if(d === 8640) {
							tim.push([rtn[0], 10080]);
							rtn[0] = 0;
							rtn[1] -= 8640;
						} else {
							rtn[1] += 1440;
						}
					}
					tim.push(rtn);
				}
				open.push(JSON.stringify(tim));
				break;

Insertion in scrape/stores.js at line 1308 [6.2622]
[6.38259]
[6.38259]
```
			tim.push(rtn);
```
Insertion in scrape/stores.js at line 1310 [6.2622]
[6.38264]
[6.38264]
```
		open.push(JSON.stringify(tim));
```
Insertion in scrape/stores.js at line 1425 [6.2622]
[6.41085]
[6.41085]
```
	let idx = [];
```

Deletion in scrape/stores.js at line 1427 [6.2622]

B:BD[6.41105] → [6.41105:41358]

	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];
	let open = [];
	let phone = [];

Deletion in scrape/stores.js at line 1429 [6.2622]
B:BD[6.41458] → [6.41458:41468]
```
		let r;
```
Insertion in scrape/stores.js at line 1430 [6.2622]
[6.41485]
[6.41485]
```
		let phone;
```

Replacement in scrape/stores.js at line 1438 [6.2622]

B:BD[6.41692] → [6.41692:41748]

						phone.push(results[i][j].replace('Phone:', ''));

[6.41692]

[6.41748]

						phone = results[i][j].replace('Phone:', '');

Insertion in scrape/stores.js at line 1519 [6.2622]

[6.44098]

		results[i] = {adr, tim, phone};
		idx.push(i);
		address.push(adr);
	}
	await postgres.query(
		`create temporary table pei as
		select *
		from unnest (
			$1::smallint[],
			$2::citext[]
		) as t(idx, address)`,
		[idx, address]
	);
	idx = (await postgres.query(`
		select idx
		from pei
		where not exists (
			select
			from store_temp
			where
				joined_name = '' and
				address = pei.address and
				address2 = '' and
				city = '' and
				region = 'pei' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);
	address = [];
	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];
	let open = [];
	let phone = [];
	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Replacement in scrape/stores.js at line 1564 [6.2622]
B:BD[6.44107] → [6.44107:44141]
```
			r = await g(`${adr} canada`);
```
[6.44107]
[6.44141]
```
			r = await g(`${results[i].adr} canada`);
```
Replacement in scrape/stores.js at line 1567 [6.2622]
B:BD[6.44179] → [6.44179:44208]
```
			console.log(results[i]);
```
[6.44179]
[6.44208]
```
			console.log(results[i].adr);
```

Replacement in scrape/stores.js at line 1571 [6.2622]

B:BD[6.44263] → [6.44263:44319]

			console.error('postal_code goofup', r, results[i]);

[6.44263]

[6.44319]

			console.error('postal_code goofup', r, results[i].adr);

Replacement in scrape/stores.js at line 1585 [6.2622]

B:BD[6.44666] → [6.44666:44701]

		open.push(JSON.stringify(tim));

[6.44666]

[6.44701]

		open.push(JSON.stringify(results[i].tim));
		phone.push(results[i].phone);

Insertion in scrape/stores.js at line 1699 [6.2622]
[6.47392]
[6.47392]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 1701 [6.2622]

[6.47517]

	let address = [];
	let results = await page.$$eval('.component-store-location__address', x => x.map(y => Array.from(y.querySelectorAll('p')).map(z => z.textContent.trim())));
	page.close();
	for(let i = 0; i < results.length; ++i) {
		idx.push(i);
		address.push(results[i][0]);
	}
	await postgres.query(
		`create temporary table yukon as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[]
		) as t(idx, joined_name, address)`,
		[idx, n, address]
	);
	idx = (await postgres.query(`
		select idx
		from yukon
		where not exists (
			select
			from store_temp
			where
				joined_name = yukon.joined_name and
				address = yukon.address and
				address2 = '' and
				city = '' and
				region = 'yukon' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);

Replacement in scrape/stores.js at line 1735 [6.2622]
B:BD[6.47535] → [6.47535:47555]
```
	let address = [];
```
[6.47535]
[6.47555]
```
	address = [];
```

Replacement in scrape/stores.js at line 1748 [6.2622]

B:BD[6.47807] → [6.47807:48025]

	let results = await page.$$eval('.component-store-location__address', x => x.map(y => Array.from(y.querySelectorAll('p')).map(z => z.textContent.trim())));
	page.close();
	for(let i = 0; i < results.length; ++i) {

[6.47807]

[6.48025]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Replacement in scrape/stores.js at line 1755 [6.2622]
B:BD[6.48145] → [6.48145:48172]
```
			console.log(names[i]);
```
[6.48145]
[6.48172]
```
			console.log(n[i], results[i]);
```

Replacement in scrape/stores.js at line 1759 [6.2622]

B:BD[6.48227] → [6.48227:48293]

			console.error('postal_code goofup', r, names[i], results[i]);

[6.48227]

[6.48293]

			console.error('postal_code goofup', r, n[i], results[i]);

Replacement in scrape/stores.js at line 1776 [6.2622]

B:BD[6.48681] → [6.48681:48883]

		for(let j = 0; j < results[i].length; ++j) {
			if(results[i][j].startsWith('http')) {
				u = results[i][j];
			} else if(results[i][j].replace(/\D/g, '').length >= 10) {
				p = results[i][j];

[6.48681]

[6.48883]

		for(let k = 1; k < results[i].length; ++k) {
			if(results[i][k].startsWith('http')) {
				u = results[i][k];
			} else if(results[i][k].replace(/\D/g, '').length === 10) {
				p = results[i][k];

Deletion in scrape/stores.js at line 1881 [6.2622]

B:BD[6.51337] → [6.51337:51611]

	let names = [];
	let address = [];
	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];
	let phone = [];

Replacement in scrape/stores.js at line 1886 [6.2622]
B:BD[6.51799] → [6.51799:51846]
```
				stuff = 'Norman Wells Liquor Agency Ltd';
```
[6.51799]
[6.51846]
```
				stuff = 'Norman Wells Liquor Agency';
```

Replacement in scrape/stores.js at line 1893 [6.2622]

B:BD[6.52027] → [6.52027:52111]

			if(stuff === 'Norman Wells Liquor Agency Ltd' && store[1] === 'Franklin Ave') {

[6.52027]

[6.52111]

			if(stuff === 'Norman Wells Liquor Agency' && store[1] === 'Franklin Ave') {

Insertion in scrape/stores.js at line 1901 [6.2622]
[6.52215]
[6.52215]
```
	let idx = [];
	let names = [];
	let address = [];
```

Insertion in scrape/stores.js at line 1905 [6.2622]

[6.52259]

		idx.push(i);
		names.push(results[i][0] = sanitize_name(results[i][0]));
		address.push(results[i][1]);
	}
	await postgres.query(
		`create temporary table nwt as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[]
		) as t(idx, joined_name, address)`,
		[idx, names, address]
	);
	idx = (await postgres.query(`
		select idx
		from nwt
		where not exists (
			select
			from store_temp
			where
				joined_name = nwt.joined_name and
				address = nwt.address and
				address2 = '' and
				city = '' and
				region = 'nwt' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];
	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];
	let phone = [];
	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Replacement in scrape/stores.js at line 1953 [6.2622]

B:BD[6.52293] → [6.52293:52353]

		if(results[i][0] === 'Hay River Liquor Retailers Ltd') {

[6.52293]

[6.52353]

		if(results[i][0] === 'Hay River Liquor') {

Replacement in scrape/stores.js at line 1974 [6.2622]
B:BD[6.52846] → [6.52846:52891]
```
		names.push(sanitize_name(results[i][0]));
```
[6.52846]
[6.52891]
```
		names.push(results[i][0]);
```
Insertion in scrape/stores.js at line 2072 [6.2622]
[6.55570]
[6.55570]
```
	let idx = [];
```

Deletion in scrape/stores.js at line 2075 [6.2622]

B:BD[6.55608] → [6.55608:55826]

	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];

Replacement in scrape/stores.js at line 2077 [6.2622]

B:BD[6.55908] → [6.55908:56369]

			let r;
			try {
				r = await g(on[i][11]);
			} catch(e) {
				console.error(e);
				console.log(on[i][6]);
				continue;
			}
			if(!pc.test(r[0].postal_code)) {
				on[i][10] = on[i][10].replace(/\s+/g, '');
				if(r[0].postal_code.length === 3 && on[i][10].length === 6) {
					r[0].postal_code = on[i][10]
				} else {
					console.error('postal_code goofup', r, on[i]);
					continue;
				}
			}
			names.push(sanitize_name(on[i][6])

[6.55908]

[6.56369]

			idx.push(i);
			names.push(on[i][6] = sanitize_name(on[i][6])

Replacement in scrape/stores.js at line 2094 [6.2622]

B:BD[6.56896] → [6.56896:57235]

			address.push(r[0].address);
			address2.push(r[0].address2);
			city.push(r[0].city);
			region.push(r[0].region);
			country.push(r[0].country);
			postal_code.push(r[0].postal_code);
			longitude.push(r[1].lng);
			latitude.push(r[1].lat);
			osrm_hint_foot.push(r[3]);
			osrm_hint_car.push(r[4]);
			timezone.push(r[2]);

[6.56896]

[6.57235]

			address.push(on[i][11]);
		}
	}
	await postgres.query(
		`create temporary table ontario as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[]
		) as t(idx, joined_name, address)`,
		[idx, names, address]
	);
	idx = (await postgres.query(`
		select idx
		from ontario
		where not exists (
			select
			from store_temp
			where
				joined_name = ontario.joined_name and
				address = ontario.address and
				address2 = '' and
				city = '' and
				region = 'ontario' and
				country = 'canada' and
				postal_code = ''
		)
	`)).rows.map(x => x.idx);
	names = [];
	address = [];
	let address2 = [];
	let city = [];
	let region = [];
	let country = [];
	let postal_code = [];
	let longitude = [];
	let latitude = [];
	let osrm_hint_foot = [];
	let osrm_hint_car = [];
	let timezone = [];
	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];
		let r;
		try {
			r = await g(on[i][11].replace(/OSPREY MIIKAN RD/i, 'OSPREY MIIKAN'));
		} catch(e) {
			console.error(e);
			console.log(on[i]);
			continue;
		}
		if(!pc.test(r[0].postal_code)) {
			on[i][10] = on[i][10].replace(/\s+/g, '');
			if(r[0].postal_code.length === 3 && on[i][10].length === 6) {
				r[0].postal_code = on[i][10];
			} else {
				console.error('postal_code goofup', r, on[i]);
				continue;
			}

Insertion in scrape/stores.js at line 2154 [6.2622]

[6.57240]

		names.push(on[i][6]);
		address.push(r[0].address);
		address2.push(r[0].address2);
		city.push(r[0].city);
		region.push(r[0].region);
		country.push(r[0].country);
		postal_code.push(r[0].postal_code);
		longitude.push(r[1].lng);
		latitude.push(r[1].lat);
		osrm_hint_foot.push(r[3]);
		osrm_hint_car.push(r[4]);
		timezone.push(r[2]);

Insertion in scrape/stores.js at line 2251 [6.2622]
[6.59634]
[6.59634]
```
	let idx = [];
```

Insertion in scrape/stores.js at line 2255 [6.2622]

[6.59692]

	let postal_code = [];
	for(let i = 0; i < results.length; ++i) {
		idx.push(i);
		address.push(results[i].Address.Line1);
		address2.push(results[i].Address.Line2 ?? '');
		city.push(results[i].Address.City);
		postal_code.push(results[i].Address.PostalCode);
	}
	await postgres.query(
		`create temporary table quebec as 
		select *
		from unnest (
			$1::smallint[],
			$2::citext[],
			$3::citext[],
			$4::citext[],
			$5::citext[]
		) as t(idx, address, address2, city, postal_code)`,
		[idx, address, address2, city, postal_code]
	);
	idx = (await postgres.query(`
		select idx
		from quebec
		where not exists (
			select
			from store_temp
			where
				joined_name = '' and
				address = quebec.address and
				address2 = quebec.address2 and
				city = quebec.city and
				region = 'quebec' and
				country = 'canada' and
				postal_code = quebec.postal_code
		)
	`)).rows.map(x => x.idx);
	address = [];
	address2 = [];
	city = [];

Replacement in scrape/stores.js at line 2296 [6.2622]
B:BD[6.59733] → [6.59733:59757]
```
	let postal_code = [];
```
[6.59733]
[6.59757]
```
	postal_code = [];
```

Replacement in scrape/stores.js at line 2304 [6.2622]

B:BD[6.59909] → [6.59909:59953]

	for(let i = 0; i < results.length; ++i) {

[6.59909]

[6.59953]

	for(let j = 0; j < idx.length; ++j) {
		const i = idx[j];

Insertion in scrape/stores.js at line 2493 [6.2622]

[6.65406]

	await postgres.query(`
		create temporary table store_temp_temp as
		select
			joined_name,
			address,
			'' as address2,
			city,
			'alberta' as region,
			'canada' as country,
			postal_code
		from alberta
		union all
		select
			joined_name,
			address,
			'',
			city,
			'manitoba',
			'canada',
			''
		from manitoba
		union all
		select
			joined_name,
			address,
			'',
			city,
			'saskatchewan',
			'canada',
			''
		from saskatchewan
		union all
		select
			'',
			address,
			'',
			'',
			'newbrunswick',
			'canada',
			''
		from newbrunswick
		union all
		select
			joined_name,
			address,
			'',
			city,
			'britishcolumbia',
			'canada',
			postal_code
		from britishcolumbia
		union all
		select
			joined_name,
			address,
			address2,
			city,
			'newfoundland',
			'canada',
			postal_code
		from newfoundland
		union all
		select
			'',
			address,
			address2,
			city,
			'novascotia',
			'canada',
			postal_code
		from novascotia
		union all
		select
			'',
			address,
			'',
			'',
			'pei',
			'canada',
			''
		from pei
		union all
		select
			joined_name,
			address,
			'',
			'',
			'yukon',
			'canada',
			''
		from yukon
		union all
		select
			joined_name,
			address,
			'',
			'',
			'nwt',
			'canada',
			''
		from nwt
		union all
		select
			joined_name,
			address,
			'',
			'',
			'ontario',
			'canada',
			''
		from ontario
		union all
		select
			'',
			address,
			address2,
			city,
			'quebec',
			'canada',
			postal_code
		from quebec
	`);
	let removeQ = (await postgres.query(`
		select *
		from store_temp
		where not exists (
			select
			from store_temp_temp
			where
				joined_name = store_temp.joined_name and
				address = store_temp.address and
				address2 = store_temp.address2 and
				city = store_temp.city and
				region = store_temp.region and
				country = store_temp.country and
				postal_code = store_temp.postal_code
		)
	`)).rows;
	if(removeQ.length) {
		console.log('human, should these be removed?', removeQ);
	}
	await postgres.query('truncate store_temp');
	await postgres.query('insert into store_temp select * from store_temp_temp');

Dependencies

In channels

Change contents

Insertion in initialize.sql at line 48 [4.17]

Insertion in scrape/common.js at line 104 [5.4897]

Insertion in scrape/stores.js at line 13 [6.2622]

Deletion in scrape/stores.js at line 16 [6.2622]

Insertion in scrape/stores.js at line 17 [6.2622]

Replacement in scrape/stores.js at line 60 [6.2622]

Replacement in scrape/stores.js at line 67 [6.2622]

Replacement in scrape/stores.js at line 74 [6.2622]

Replacement in scrape/stores.js at line 86 [6.2622]

Insertion in scrape/stores.js at line 206 [6.2622]

Deletion in scrape/stores.js at line 210 [6.2622]

Insertion in scrape/stores.js at line 211 [6.2622]

Replacement in scrape/stores.js at line 262 [6.2622]

Deletion in scrape/stores.js at line 293 [6.2622]

Deletion in scrape/stores.js at line 294 [6.2622]

Insertion in scrape/stores.js at line 295 [6.2622]

Insertion in scrape/stores.js at line 406 [6.2622]

Deletion in scrape/stores.js at line 409 [6.2622]

Insertion in scrape/stores.js at line 410 [6.2622]

Replacement in scrape/stores.js at line 456 [6.2622]

Replacement in scrape/stores.js at line 472 [6.2622]

Replacement in scrape/stores.js at line 496 [6.2622]

Replacement in scrape/stores.js at line 503 [6.2622]

Insertion in scrape/stores.js at line 645 [6.2622]

Insertion in scrape/stores.js at line 647 [6.2622]

Replacement in scrape/stores.js at line 688 [6.2622]

Replacement in scrape/stores.js at line 718 [6.2622]

Replacement in scrape/stores.js at line 722 [6.2622]

Insertion in scrape/stores.js at line 842 [6.2622]

Insertion in scrape/stores.js at line 845 [6.2622]

Replacement in scrape/stores.js at line 887 [6.2622]

Replacement in scrape/stores.js at line 890 [6.2622]

Replacement in scrape/stores.js at line 897 [6.2622]

Deletion in scrape/stores.js at line 915 [6.2622]

Insertion in scrape/stores.js at line 916 [6.2622]

Insertion in scrape/stores.js at line 1014 [6.2622]

Insertion in scrape/stores.js at line 1019 [6.2622]

Replacement in scrape/stores.js at line 1063 [6.2622]

Replacement in scrape/stores.js at line 1071 [6.2622]

Replacement in scrape/stores.js at line 1078 [6.2622]

Replacement in scrape/stores.js at line 1091 [6.2622]

Insertion in scrape/stores.js at line 1194 [6.2622]

Insertion in scrape/stores.js at line 1198 [6.2622]

Replacement in scrape/stores.js at line 1244 [6.2622]

Replacement in scrape/stores.js at line 1252 [6.2622]

Deletion in scrape/stores.js at line 1307 [6.2622]

Insertion in scrape/stores.js at line 1308 [6.2622]

Insertion in scrape/stores.js at line 1310 [6.2622]

Insertion in scrape/stores.js at line 1425 [6.2622]

Deletion in scrape/stores.js at line 1427 [6.2622]

Deletion in scrape/stores.js at line 1429 [6.2622]

Insertion in scrape/stores.js at line 1430 [6.2622]

Replacement in scrape/stores.js at line 1438 [6.2622]

Insertion in scrape/stores.js at line 1519 [6.2622]

Replacement in scrape/stores.js at line 1564 [6.2622]

Replacement in scrape/stores.js at line 1567 [6.2622]

Replacement in scrape/stores.js at line 1571 [6.2622]

Replacement in scrape/stores.js at line 1585 [6.2622]

Insertion in scrape/stores.js at line 1699 [6.2622]

Insertion in scrape/stores.js at line 1701 [6.2622]

Replacement in scrape/stores.js at line 1735 [6.2622]

Replacement in scrape/stores.js at line 1748 [6.2622]

Replacement in scrape/stores.js at line 1755 [6.2622]