1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
const http = require( 'http' );
const fs = require( 'fs' );
const path = require( 'path' );
const crypto = require( 'crypto' );
const puppeteer = require( 'puppeteer' );
const express = require( 'express' );
const PORT = 6860;
const CACHE_DIR = path.join( __dirname, 'cache' );
const host = "https://networkheaven.net";
const app = express();
if( !fs.existsSync( CACHE_DIR ) ) fs.mkdirSync( CACHE_DIR );
function hash( str ) {
return crypto.createHash( 'sha1' ).update( str ).digest( 'hex' );
}
let browser = null;
let cache = async ( req, res ) => {
const { url } = req;
console.log( `incoming req from ${req.headers['user-agent']} for ${url}` );
const urlObj = new URL( req.url, host );
const target = urlObj.href;
const key = hash( target );
const file = path.join( CACHE_DIR, key + '.html' );
if( fs.existsSync( file ) ) {
const html = fs.readFileSync( file );
res.writeHead( 200, { 'content-type': 'text/html' } );
return res.end( html );
}
try {
const page = await browser.newPage();
const r = await page.goto( target, {
waitUntil: 'networkidle2',
timeout: 30000
} );
const html = await page.content();
console.log( "caching page: " + target );
fs.writeFileSync( file, html );
await page.close();
const headers = r.headers();
res.writeHead( 200, {
'content-type': headers['content-type'],
'date': headers['date'],
'etag': headers['etag'],
} );
res.end( html );
} catch( e ) {
res.writeHead( 500, { 'content-type': 'text/plain' } );
res.end( 'render failed: ' + e.message );
}
}
app.get( "/", cache );
app.get( /\/$/, cache );
app.get( /#$/, cache );
app.get( /^\/[^.]*$/, cache );
app.use( async ( req, res ) => {
let url = new URL( req.url, host );
const r = await fetch( url.href );
if( r.ok ) {
const headers = [];
for( const [k, v] of r.headers.entries() )
headers[k] = v;
res.writeHead( 200, {
'content-type': headers['content-type'],
'date': headers['date'],
'etag': headers['etag'],
} );
if( url.href.endsWith( ".ico" ) ) {
const ab = await r.arrayBuffer();
return res.end( Buffer.from( ab ) );
}
const html = await r.text();
res.end( html );
}
} );
(async () => {
browser = await puppeteer.launch( { headless: 'new' } );
app.listen( PORT, async () => {
console.log( "listening on http://localhost:" + PORT );
} );
})();
|
