new blogposts
This commit is contained in:
parent
7bb00ab2c0
commit
71e0905acd
Binary file not shown.
Binary file not shown.
|
|
@ -25,77 +25,55 @@
|
|||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3>all posts</h3>
|
||||
<h4 class='allposts_header'>September 2018</h4>
|
||||
<h4 class='allposts_header'>November 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./italy.html">italy</a> — September 20, 2018</li>
|
||||
<li><a href="./utterances.html">utterances</a> — September 05, 2018</li>
|
||||
<li><a href="./proactive-redundancy.html">proactive redundancy</a> — November 15, 2018</li>
|
||||
<li><a href="./november-13-post-mortem.html">november 13 post mortem</a> — November 13, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>August 2018</h4>
|
||||
<h4 class='allposts_header'>October 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./no-more-google.html">no more google</a> — August 14, 2018</li>
|
||||
<li><a href="./upsides-of-new-dns-nameservers.html">upsides of new dns nameservers</a> — August 14, 2018</li>
|
||||
<li><a href="./dns-shenanigans-post-mortem.html">dns shenanigans post-mortem</a> — August 14, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>July 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./lxd-networking-and-additional-ips.html">lxd networking and additional IPs</a> — July 26, 2018</li>
|
||||
<li><a href="./dotfiles.html">dotfiles</a> — July 22, 2018</li>
|
||||
<li><a href="./bashblog-and-your-gopherhole.html">bashblog and your gopherhole</a> — July 22, 2018</li>
|
||||
<li><a href="./more-drone-photos.html">more drone photos</a> — July 15, 2018</li>
|
||||
<li><a href="./tildeverseorg.html">tildeverse.org</a> — July 15, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>June 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./tildeteam-news.html">tilde.team news</a> — June 13, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>March 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./white-pride-vs-black-pride.html">white pride vs black pride</a> — March 07, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>February 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./phoenix.html">phoenix</a> — February 26, 2018</li>
|
||||
<li><a href="./otm.html">otm</a> — February 15, 2018</li>
|
||||
<li><a href="./quote-of-the-day.html">quote of the day</a> — February 13, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>January 2018</h4>
|
||||
<ul>
|
||||
<li><a href="./webassembly.html">webassembly</a> — January 17, 2018</li>
|
||||
<li><a href="./pop-quiz.html">pop quiz</a> — January 16, 2018</li>
|
||||
<li><a href="./git-remotes-with-ssh-aliases.html">git remotes with ssh aliases</a> — January 12, 2018</li>
|
||||
<li><a href="./cold.html">cold</a> — January 05, 2018</li>
|
||||
<li><a href="./8values.html">8values</a> — January 03, 2018</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>December 2017</h4>
|
||||
<ul>
|
||||
<li><a href="./mastodon.html">mastodon</a> — December 22, 2017</li>
|
||||
<li><a href="./loading.html">loading...</a> — December 21, 2017</li>
|
||||
<li><a href="./vr.html">vr</a> — December 18, 2017</li>
|
||||
<li><a href="./net-neutrality-vote-today.html">net neutrality vote today</a> — December 14, 2017</li>
|
||||
<li><a href="./hey-dere-bub.html">hey dere bub!</a> — December 13, 2017</li>
|
||||
<li><a href="./pan-galactic-gargle-blaster.html">pan galactic gargle blaster</a> — December 07, 2017</li>
|
||||
<li><a href="./dont-be-a-coconut.html">don't be a coconut</a> — December 05, 2017</li>
|
||||
<li><a href="./thought-of-the-day2227.html">thought of the day</a> — December 03, 2017</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>November 2017</h4>
|
||||
<ul>
|
||||
<li><a href="./where-to-find-me-elsewhere-on-the-web.html">where to find me elsewhere on the web</a> — November 28, 2017</li>
|
||||
<li><a href="./blog-update.html">blog update</a> — November 27, 2017</li>
|
||||
<li><a href="./thought-of-the-day27904.html">thought of the day</a> — November 27, 2017</li>
|
||||
<li><a href="./antiwitze.html">antiwitze</a> — November 27, 2017</li>
|
||||
<li><a href="./thought-of-the-day14302.html">thought of the day</a> — November 27, 2017</li>
|
||||
<li><a href="./nonsense.html">Nonsense</a> — November 27, 2017</li>
|
||||
<li><a href="./thought-of-the-day22873.html">Thought of the Day</a> — November 27, 2017</li>
|
||||
<li><a href="./christian-morgenstern---verkehrte-welt.html">Christian Morgenstern – “verkehrte Welt”</a> — November 27, 2017</li>
|
||||
<li><a href="./joe-on-sporty-ball-z.html">Joe on Sporty-ball-z</a> — November 27, 2017</li>
|
||||
<li><a href="./fun-words-in-german.html">fun words in german</a> — November 27, 2017</li>
|
||||
<li><a href="./thought-of-the-day.html">Thought of the day</a> — November 27, 2017</li>
|
||||
</ul>
|
||||
<h4 class='allposts_header'>October 2017</h4>
|
||||
<ul>
|
||||
<li><a href="./links-to-save-for-later.html">links to save for later</a> — October 20, 2017</li>
|
||||
<li><a href="./hi-there.html">hi there</a> — October 02, 2017</li>
|
||||
<li><a href="./4k-gaming-with-a-gtx1080ti.html">4k gaming with a gtx1080ti</a> — October 02, 2017</li>
|
||||
<li><a href="./quote-of-the-day.html">quote of the day</a> — October 23, 2018</li>
|
||||
<li><a href="./thought-of-the-day14302.html">thought of the day</a> — October 23, 2018</li>
|
||||
<li><a href="./thought-of-the-day2227.html">thought of the day</a> — October 23, 2018</li>
|
||||
<li><a href="./thought-of-the-day22873.html">Thought of the Day</a> — October 23, 2018</li>
|
||||
<li><a href="./thought-of-the-day27904.html">thought of the day</a> — October 23, 2018</li>
|
||||
<li><a href="./thought-of-the-day.html">Thought of the day</a> — October 23, 2018</li>
|
||||
<li><a href="./tildeteam-news.html">tilde.team news</a> — October 23, 2018</li>
|
||||
<li><a href="./tildeverseorg.html">tildeverse.org</a> — October 23, 2018</li>
|
||||
<li><a href="./upsides-of-new-dns-nameservers.html">upsides of new dns nameservers</a> — October 23, 2018</li>
|
||||
<li><a href="./utterances.html">utterances</a> — October 23, 2018</li>
|
||||
<li><a href="./vr.html">vr</a> — October 23, 2018</li>
|
||||
<li><a href="./webassembly.html">webassembly</a> — October 23, 2018</li>
|
||||
<li><a href="./where-to-find-me-elsewhere-on-the-web.html">where to find me elsewhere on the web</a> — October 23, 2018</li>
|
||||
<li><a href="./white-pride-vs-black-pride.html">white pride vs black pride</a> — October 23, 2018</li>
|
||||
<li><a href="./4k-gaming-with-a-gtx1080ti.html">4k gaming with a gtx1080ti</a> — October 23, 2018</li>
|
||||
<li><a href="./8values.html">8values</a> — October 23, 2018</li>
|
||||
<li><a href="./antiwitze.html">antiwitze</a> — October 23, 2018</li>
|
||||
<li><a href="./bashblog-and-your-gopherhole.html">bashblog and your gopherhole</a> — October 23, 2018</li>
|
||||
<li><a href="./blog-update.html">blog update</a> — October 23, 2018</li>
|
||||
<li><a href="./christian-morgenstern---verkehrte-welt.html">Christian Morgenstern – “verkehrte Welt”</a> — October 23, 2018</li>
|
||||
<li><a href="./cold.html">cold</a> — October 23, 2018</li>
|
||||
<li><a href="./dns-shenanigans-post-mortem.html">dns shenanigans post-mortem</a> — October 23, 2018</li>
|
||||
<li><a href="./dont-be-a-coconut.html">don't be a coconut</a> — October 23, 2018</li>
|
||||
<li><a href="./dotfiles.html">dotfiles</a> — October 23, 2018</li>
|
||||
<li><a href="./fun-words-in-german.html">fun words in german</a> — October 23, 2018</li>
|
||||
<li><a href="./git-remotes-with-ssh-aliases.html">git remotes with ssh aliases</a> — October 23, 2018</li>
|
||||
<li><a href="./hey-dere-bub.html">hey dere bub!</a> — October 23, 2018</li>
|
||||
<li><a href="./hi-there.html">hi there</a> — October 23, 2018</li>
|
||||
<li><a href="./italy.html">italy</a> — October 23, 2018</li>
|
||||
<li><a href="./joe-on-sporty-ball-z.html">Joe on Sporty-ball-z</a> — October 23, 2018</li>
|
||||
<li><a href="./links-to-save-for-later.html">links to save for later</a> — October 23, 2018</li>
|
||||
<li><a href="./loading.html">loading...</a> — October 23, 2018</li>
|
||||
<li><a href="./lxd-networking-and-additional-ips.html">lxd networking and additional IPs</a> — October 23, 2018</li>
|
||||
<li><a href="./mastodon.html">mastodon</a> — October 23, 2018</li>
|
||||
<li><a href="./more-drone-photos.html">more drone photos</a> — October 23, 2018</li>
|
||||
<li><a href="./net-neutrality-vote-today.html">net neutrality vote today</a> — October 23, 2018</li>
|
||||
<li><a href="./no-more-google.html">no more google</a> — October 23, 2018</li>
|
||||
<li><a href="./nonsense.html">Nonsense</a> — October 23, 2018</li>
|
||||
<li><a href="./otm.html">otm</a> — October 23, 2018</li>
|
||||
<li><a href="./pan-galactic-gargle-blaster.html">pan galactic gargle blaster</a> — October 23, 2018</li>
|
||||
<li><a href="./phoenix.html">phoenix</a> — October 23, 2018</li>
|
||||
<li><a href="./pop-quiz.html">pop quiz</a> — October 23, 2018</li>
|
||||
</ul>
|
||||
<div id="all_posts"><a href="./index.html">back home</a></div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@
|
|||
<li><a href="tag_jokes.html">jokes</a> — 2 posts</li>
|
||||
<li><a href="tag_linguistics.html">linguistics</a> — 1 post</li>
|
||||
<li><a href="tag_links.html">links</a> — 1 post</li>
|
||||
<li><a href="tag_linux.html">linux</a> — 6 posts</li>
|
||||
<li><a href="tag_linux.html">linux</a> — 7 posts</li>
|
||||
<li><a href="tag_lyrics.html">lyrics</a> — 1 post</li>
|
||||
<li><a href="tag_mastodon.html">mastodon</a> — 1 post</li>
|
||||
<li><a href="tag_music.html">music</a> — 1 post</li>
|
||||
|
|
@ -60,12 +60,13 @@
|
|||
<li><a href="tag_podcast.html">podcast</a> — 2 posts</li>
|
||||
<li><a href="tag_poetry.html">poetry</a> — 1 post</li>
|
||||
<li><a href="tag_politics.html">politics</a> — 1 post</li>
|
||||
<li><a href="tag_post-mortem.html">post-mortem</a> — 1 post</li>
|
||||
<li><a href="tag_save-for-later.html">save-for-later</a> — 1 post</li>
|
||||
<li><a href="tag_snow.html">snow</a> — 1 post</li>
|
||||
<li><a href="tag_social-networks.html">social-networks</a> — 1 post</li>
|
||||
<li><a href="tag_ssh.html">ssh</a> — 1 post</li>
|
||||
<li><a href="tag_sysadmin.html">sysadmin</a> — 4 posts</li>
|
||||
<li><a href="tag_tilde.html">tilde</a> — 6 posts</li>
|
||||
<li><a href="tag_sysadmin.html">sysadmin</a> — 6 posts</li>
|
||||
<li><a href="tag_tilde.html">tilde</a> — 7 posts</li>
|
||||
<li><a href="tag_travel.html">travel</a> — 1 post</li>
|
||||
<li><a href="tag_ubuntu.html">ubuntu</a> — 2 posts</li>
|
||||
<li><a href="tag_update.html">update</a> — 1 post</li>
|
||||
|
|
|
|||
1828
blog/feed.rss
1828
blog/feed.rss
File diff suppressed because it is too large
Load Diff
508
blog/index.html
508
blog/index.html
|
|
@ -24,260 +24,326 @@
|
|||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3><a class="ablack" href="italy.html">
|
||||
italy
|
||||
<h3><a class="ablack" href="proactive-redundancy.html">
|
||||
proactive redundancy
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201809201732.33# -->
|
||||
<div class="subtitle">September 20, 2018 —
|
||||
<!-- bashblog_timestamp: #201811151839.26# -->
|
||||
<div class="subtitle">November 15, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i just got back from a 10-day backpacking trip to italy and i'd like to share some of the photos i took!</p>
|
||||
<p>after the <a href="november-13-post-mortem.html">fiasco</a> earlier this week, i've been taking steps to minimize
|
||||
the impact if tilde.team were to go down. it's still a large spof (single-point-of-failure), but i'm reasonably certain that at least the irc net will remain up and functional in the event of another outage. </p>
|
||||
|
||||
<p>the travel plan was rome -> venice -> florence -> naples -> pompei/vesuvius -> capri -> amalfi</p>
|
||||
<p>the first thing that i set up was a handful of additional ircd nodes: see <a href="https://tilde.chat/wiki/?page=servers">the tilde.chat wiki</a> for a full list. slash.tilde.chat is on my personal vps, and bsd.tilde.chat is hosted on the bsd vps that i set up for tilde.team. </p>
|
||||
|
||||
<p>this is the roman forum (with colosseum in the background) as seen from the palatine.</p>
|
||||
<p>i added the ipv4 addresses for these machines, along with the ip for yourtilde.com as A records for tilde.chat, creating a dns round-robin. <code>host tilde.chat</code> will return all four. requesting the dns record will return any one of them, rotating them in a semi-random fashion. this means that when connecting to tilde.chat on 6697 for irc, you might end up on any of <code>{your,team,bsd,slash}.tilde.chat</code>. </p>
|
||||
|
||||
<p><img src="https://bhh.sh/pub/photos/italy/roman-forum.jpg" alt="" title="" /></p>
|
||||
<p>this creates the additional problem that visiting the <a href="https://tilde.chat">tilde.chat site</a> will end up at any of those 4 machines in much the same way. for the moment, the site is deployed on all of the boxes, making site setup issues hard to <a href="https://tildegit.org/tildeverse/tilde.chat/issues/8">debug</a>. the solution to this problem is to use a subdomain as the roundrobin host, as other networks like freenode do (see <code>host chat.freenode.net</code> for the list of servers).</p>
|
||||
|
||||
<p class="readmore"><a href="./italy.html">read more...</a></p>
|
||||
<h3><a class="ablack" href="utterances.html">
|
||||
utterances
|
||||
<p>i'm not sure how to make any of the other services more resilient. it's something that i have been and will continue to research moving forward.</p>
|
||||
|
||||
<p>the other main step that i have taken to prevent the same issue from happening again was to configure the firewall to drop outgoing requests to the subnets as defined in <a href="https://tools.ietf.org/html/rfc1918">rfc 1918</a>.</p>
|
||||
|
||||
<p>i'd like to consider at least this risk to be mitigated.</p>
|
||||
|
||||
<p>thanks for reading,</p>
|
||||
|
||||
<p>~ben</p>
|
||||
|
||||
<p>tags: <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="november-13-post-mortem.html">
|
||||
november 13 post mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201809052134.13# -->
|
||||
<div class="subtitle">September 05, 2018 —
|
||||
<!-- bashblog_timestamp: #201811132020.33# -->
|
||||
<div class="subtitle">November 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i somehow stumbled upon <a href="https://utteranc.es">utterances</a> today at lunch. (i think someone had it forked on their github page).</p>
|
||||
<p>we had something of an outage on november 13, 2018 on tilde.team. </p>
|
||||
|
||||
<p>i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.</p>
|
||||
|
||||
<p>tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.</p>
|
||||
|
||||
<blockquote>
|
||||
<p>We have indications that there was an attack from your server.
|
||||
Please take all necessary measures to avoid this in the future and to solve the issue.</p>
|
||||
</blockquote>
|
||||
|
||||
<p>at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.</p>
|
||||
|
||||
<p>when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the <a href="https://mail.tilde.team">webmail</a> to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!</p>
|
||||
|
||||
<p>here, i launch in to full debugging mode: what command was it? who ran it? </p>
|
||||
|
||||
<p>search <code>~/.bash_history</code> per user was not very successful. nothing i could find was related to net or map. i had checked <code>sudo grep nmap /home/*/.bash_history</code> and many other commands. </p>
|
||||
|
||||
<p>at this point, i had connected with other ~teammates across other irc nets (<a href="https://hashbang.sh/">#!</a>, <a href="https://tilde.town">~town</a>, etc). among suggestions to check <code>/var/log/syslog</code>, <code>/var/log/kern.log</code>, and <code>dmesg</code>, i finally decided to check <code>ps</code>. <code>ps -ef | grep nmap</code> yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for <a href="/~fosslinux/">~fosslinux</a>.</p>
|
||||
|
||||
<p>i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police <code>nmap</code> when it isn't scanning on every port?</p>
|
||||
|
||||
<p>after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that <a href="/~fosslinux/">~fosslinux</a> had only run <code>nmap</code> for addresses in the <code>10.0.0.0/8</code> space. the <code>10/8</code> address space is intended to not be addressable outside the local space. how could <a href="https://hetzner.com">hetzner</a> have found out about a localhost network probe!?</p>
|
||||
|
||||
<p>finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.</p>
|
||||
|
||||
<p>it's definitely time to research redundancy options!</p>
|
||||
|
||||
<p>tags: <a href='tag_post-mortem.html'>post-mortem</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="quote-of-the-day.html">
|
||||
quote of the day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201802130955.06# -->
|
||||
<div class="subtitle">February 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>Be Alert! - the world needs more Lerts.</p>
|
||||
|
||||
<p>Tags: <a href='tag_quotes.html'>quotes</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<p>no matter how i found it, i still decided to add it to my blog here with <a href="https://tildegit.org/team/bashblog">bashblog</a>. utterances is a commenting system that leverages github issues. so, for example a comment on <a href="https://tilde.team/~ben/blog/upsides-of-new-dns-nameservers.html">a post</a> shows up on github <a href="https://github.com/benharri/tilde/issues/1#issuecomment-418732788">like this</a>.</p>
|
||||
|
||||
<p>now we just need to figure out if it can be pointed at a gitea instance like <a href="https://tildegit.org">tildegit</a>. might be time for a PR!</p>
|
||||
|
||||
<p>tags: <a href='tag_blog.html'>blog</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="no-more-google.html">
|
||||
no more google
|
||||
<h3><a class="ablack" href="thought-of-the-day14302.html">
|
||||
thought of the day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201808142336.05# -->
|
||||
<div class="subtitle">August 14, 2018 —
|
||||
<!-- bashblog_timestamp: #201711271656.36# -->
|
||||
<div class="subtitle">November 27, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>why do they tell us to use the stairs in case of fire? shouldn't we be using a fire extinguisher?</p>
|
||||
|
||||
<p>Tags: <a href='tag_thought-of-the-day.html'>thought-of-the-day</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="thought-of-the-day2227.html">
|
||||
thought of the day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201712031347.36# -->
|
||||
<div class="subtitle">December 03, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>everything in the universe either is or isn't a potato.</p>
|
||||
|
||||
<p>Tags: <a href='tag_thought-of-the-day.html'>thought-of-the-day</a>, <a href='tag_words.html'>words</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="thought-of-the-day22873.html">
|
||||
Thought of the Day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201711271654.07# -->
|
||||
<div class="subtitle">November 27, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>“Arguing with religious people – It’s like playing chess with a pigeon; no matter how good I am at chess, the pigeon is just going to knock over the pieces, crap on the board and strut around victorious” – Anonymous</p>
|
||||
|
||||
<p>Tags: <a href='tag_nonsense.html'>nonsense</a>, <a href='tag_quotes.html'>quotes</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="thought-of-the-day27904.html">
|
||||
thought of the day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201711271658.50# -->
|
||||
<div class="subtitle">November 27, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>wherever you go, there you are</p>
|
||||
|
||||
<p>Tags: <a href='tag_thought-of-the-day.html'>thought-of-the-day</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="thought-of-the-day.html">
|
||||
Thought of the day
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201711271649.29# -->
|
||||
<div class="subtitle">November 27, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>things are not what they appear to be. nor are they otherwise.</p>
|
||||
|
||||
<p>Tags: <a href='tag_thought-of-the-day.html'>thought-of-the-day</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="tildeteam-news.html">
|
||||
tilde.team news
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201806131507.45# -->
|
||||
<div class="subtitle">June 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>not sure if this is appropriately tagged, but i didn't feel like making a new
|
||||
one.</p>
|
||||
<p>hey hi hello!</p>
|
||||
|
||||
<p>i figured i should probably get some notes down about moving off google.</p>
|
||||
<p>it seems that i haven't written anything on my blog in quite a while...</p>
|
||||
|
||||
<p>to start, i'll get a list of the things i was able to easily replace:</p>
|
||||
<p>time to fix that! i've been quite busy in the last month or so with a lot of new ideas an energy for tilde.team.</p>
|
||||
|
||||
<p>after rediscovering my account on tilde.town, i hopped in the irc there and my enthusiasm translated into a couple new members over here on the ~team.</p>
|
||||
|
||||
<p>our irc has been somewhat more active recently which is awesome:)</p>
|
||||
|
||||
<p>some of the new updates in the last month:</p>
|
||||
|
||||
<ul>
|
||||
<li>gmail => <a href="https://tilde.team/wiki/?page=email">@tilde.team mail</a></li>
|
||||
<li>google drive => <a href="https://syncthing.net">syncthing</a> (with a persistent node running on my personal vps)</li>
|
||||
<li><a href="https://git.tilde.team">tildegit (our own gitea instance)</a></li>
|
||||
<li><a href="https://mail.tilde.team">tildemail</a> with postfix and dovecot for smtp/imap as well as local command line mail in mutt and alpine</li>
|
||||
<li><a href="https://git.tildeverse.org/team/tilde-launcher"><code>tilde</code></a> user script wrapper with submission and approval flows</li>
|
||||
<li><a href="https://tilde.team/wiki/?page=ssh">password auth disabled</a></li>
|
||||
</ul>
|
||||
|
||||
<p>i'm still using:</p>
|
||||
|
||||
<ul>
|
||||
<li>gplay music/youtube</li>
|
||||
<li>google maps (open streetmap isn't good enough to replace it)</li>
|
||||
<li>google photos - but this is going to be replaced long-term with syncthing</li>
|
||||
</ul>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_net-neutrality.html'>net-neutrality</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="upsides-of-new-dns-nameservers.html">
|
||||
upsides of new dns nameservers
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201808141505.38# -->
|
||||
<div class="subtitle">August 14, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<ul>
|
||||
<li>no more google</li>
|
||||
<li>no more google</li>
|
||||
<li>automated certbot validation for letsencrypt wildcard certs!! no more manual TXT records every three months!</li>
|
||||
</ul>
|
||||
|
||||
<p>tags: <a href='tag_dns.html'>dns</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="dns-shenanigans-post-mortem.html">
|
||||
dns shenanigans post-mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201808141503.49# -->
|
||||
<div class="subtitle">August 14, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>let's start by saying i probably should have done a bit more research before
|
||||
diving head-first into this endeavor.</p>
|
||||
|
||||
<p>i've been thinking about transferring my domains off google domains for some
|
||||
time now, as part of my personal goal to self host and limit my dependence on
|
||||
google and other large third-party monstrosities. along that line, i asked for
|
||||
registrar recommendations. <a href="https://tomasino.tilde.team">~tomasino</a> responded
|
||||
with <a href="https://namesilo.com">namesilo</a>. i found that they had $3.99 registrations
|
||||
for .team and .zone domains, which is 1/10th the cost of the $40 registration
|
||||
on google domains.</p>
|
||||
|
||||
<p>i started out by getting the list of domains from the google console. 2 or 3
|
||||
of them had been registered within the last 60 days, so i wasn't able to
|
||||
transfer those just yet. i grabbed all the domain unlock codes and dropped
|
||||
them into namesilo. i failed to realize that the dns panel on google domains
|
||||
would disappear as soon as it went through, but more importantly that the
|
||||
nameservers would be left pointing to the old defunct google domains ones.</p>
|
||||
|
||||
<p>i updated the nameservers as soon as i realized this error from the namesilo
|
||||
panel. some of the domains propagated quickly. others, not so much. tilde.team
|
||||
was still in a state of flux between the old and new nameservers.</p>
|
||||
|
||||
<p>in a rush to get the dns problem fixed, and under recommendation from several
|
||||
people on irc, i decided to switch the nameservers for tilde.team and tilde.zone
|
||||
to cloudflare, leaving another layer of flux for the dns to be stuck in...</p>
|
||||
|
||||
<p>of the five domains that i moved to cloudflare, 3 returned with a dnssec error,
|
||||
claiming that i needed to remove the DS record from that zone. d'oh!</p>
|
||||
|
||||
<p>i removed the dnssec from those affected domains, so we should be good to go
|
||||
as soon as it all propagates through the fickle beast that is dns.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_tilde.html'>tilde</a>, <a href='tag_dns.html'>dns</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="lxd-networking-and-additional-ips.html">
|
||||
lxd networking and additional IPs
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807261534.50# -->
|
||||
<div class="subtitle">July 26, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>now that tilde.team is on a fancy-shmancy new dedi server, i've tried to get a secondary IP address
|
||||
assigned to a lxd container (which i plan to use for my personal stuff). lxd shows that the secondary
|
||||
IP is being picked up by that container, but i'm still seeing the host machine's IP as the external
|
||||
address.</p>
|
||||
|
||||
<p>i'm not sure how i'll need to configure the network settings on the host machine (now that we're running
|
||||
ubuntu 18.04 and it uses netplan for configs and not /etc/network/interfaces). another confusing thing is
|
||||
that the main config in /etc/netplan says that the network config is handled by systemd-networkd...</p>
|
||||
|
||||
<p>at least i have through the end of the year when my current vps runs out to get this up and running.</p>
|
||||
|
||||
<p>ping me on <a href="https://tilde.chat">irc</a> or <a href="mailto:ben@tilde.team">email</a> if you have experience with this.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_ubuntu.html'>ubuntu</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="dotfiles.html">
|
||||
dotfiles
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807221926.26# -->
|
||||
<div class="subtitle">July 22, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>finally got around to updating my <a href="https://git.tilde.team/ben/dotfiles">dotfiles</a> to use gnu stow.
|
||||
i adapted <a href="https://github.com/jamestomasino/dotfiles/blob/master/Makefile">~tomasino's makefile</a>
|
||||
for use with the configs that i'm keeping with it.</p>
|
||||
|
||||
<p>now i just need to figure out why my ssh config doesn't copy/symlink my config to ~/.ssh when it
|
||||
already exists.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_dotfiles.html'>dotfiles</a>, <a href='tag_git.html'>git</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="bashblog-and-your-gopherhole.html">
|
||||
bashblog and your gopherhole
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807221144.03# -->
|
||||
<div class="subtitle">July 22, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i've created <a href="https://git.tildeverse.org/meta/bashblog">a repo</a> for the tilde.team customizations to <a href="https://github.com/cfenollosa/bashblog">bashblog</a>.</p>
|
||||
|
||||
<p>it will now make sure that your ~/public_gopher exists and symlink your blog into it with a nice gophermap to list all the markdown styled posts.</p>
|
||||
|
||||
<p>try it out and let me know if there are any problems!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_blog.html'>blog</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="more-drone-photos.html">
|
||||
more drone photos
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807152315.46# -->
|
||||
<div class="subtitle">July 15, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i finally got my drone out this summer to take some more pics!</p>
|
||||
|
||||
<p><img src="https://bhh.sh/pub/photos/drone/DJI_0097.thumb.jpg" alt="" title="" /></p>
|
||||
|
||||
<p><a href="https://bhh.sh/pub/photos/drone/">more here</a></p>
|
||||
|
||||
<p>tags: <a href='tag_dji.html'>dji</a>, <a href='tag_drone.html'>drone</a>, <a href='tag_photography.html'>photography</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<p>i'd like to make use of our new mailserver, so shoot me some <a href="mailto:ben@tilde.team">mail</a>.
|
||||
i never get enough personal mail. it's all still privacy policy update notices. :(</p>
|
||||
|
||||
<p>see you soon!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
|
||||
<meta name="theme-color" content="#00cc00">
|
||||
<link rel="icon" type="image/png" sizes="192x192" href="https://tilde.team/apple-touch-icon-precomposed.png">
|
||||
<link rel="icon" type="image/png" sizes="96x96" href="https://tilde.team/favicon-96x96.png">
|
||||
|
||||
<link rel="stylesheet" href="https://tilde.team/css/hacker.css">
|
||||
<link rel="stylesheet" href="extra.css">
|
||||
|
||||
<link rel="alternate" type="application/rss+xml" title="subscribe to this page..." href="feed.rss" />
|
||||
<title>november 13 post mortem</title>
|
||||
</head><body>
|
||||
<div class="container">
|
||||
|
||||
<div id="divbodyholder">
|
||||
<div class="headerholder"><div class="header">
|
||||
<div id="title">
|
||||
<h1 class="nomargin"><a class="ablack" href="https://tilde.team/~ben/blog/index.html">blog // ~ben</a></h1>
|
||||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<!-- entry begin -->
|
||||
<h3><a class="ablack" href="november-13-post-mortem.html">
|
||||
november 13 post mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811132020.33# -->
|
||||
<div class="subtitle">November 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>we had something of an outage on november 13, 2018 on tilde.team. </p>
|
||||
|
||||
<p>i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.</p>
|
||||
|
||||
<p>tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.</p>
|
||||
|
||||
<blockquote>
|
||||
<p>We have indications that there was an attack from your server.
|
||||
Please take all necessary measures to avoid this in the future and to solve the issue.</p>
|
||||
</blockquote>
|
||||
|
||||
<p>at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.</p>
|
||||
|
||||
<p>when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the <a href="https://mail.tilde.team">webmail</a> to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!</p>
|
||||
|
||||
<p>here, i launch in to full debugging mode: what command was it? who ran it? </p>
|
||||
|
||||
<p>search <code>~/.bash_history</code> per user was not very successful. nothing i could find was related to net or map. i had checked <code>sudo grep nmap /home/*/.bash_history</code> and many other commands. </p>
|
||||
|
||||
<p>at this point, i had connected with other ~teammates across other irc nets (<a href="https://hashbang.sh/">#!</a>, <a href="https://tilde.town">~town</a>, etc). among suggestions to check <code>/var/log/syslog</code>, <code>/var/log/kern.log</code>, and <code>dmesg</code>, i finally decided to check <code>ps</code>. <code>ps -ef | grep nmap</code> yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for <a href="/~fosslinux/">~fosslinux</a>.</p>
|
||||
|
||||
<p>i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police <code>nmap</code> when it isn't scanning on every port?</p>
|
||||
|
||||
<p>after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that <a href="/~fosslinux/">~fosslinux</a> had only run <code>nmap</code> for addresses in the <code>10.0.0.0/8</code> space. the <code>10/8</code> address space is intended to not be addressable outside the local space. how could <a href="https://hetzner.com">hetzner</a> have found out about a localhost network probe!?</p>
|
||||
|
||||
<p>finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.</p>
|
||||
|
||||
<p>it's definitely time to research redundancy options!</p>
|
||||
|
||||
<p>tags: <a href='tag_post-mortem.html'>post-mortem</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
<!-- text end -->
|
||||
<!-- entry end -->
|
||||
</div>
|
||||
<div id="footer">CC by-nc-nd <a href="https://tilde.team/~ben/">~ben</a> — <a href="mailto:ben@tilde.team">ben@tilde.team</a><br/>
|
||||
generated with <a href="https://tildegit.org/team/bashblog">bashblog</a>, a single bash script to easily create blogs like this one</div>
|
||||
</div></div>
|
||||
<script src="https://utteranc.es/client.js"
|
||||
repo="benharri/tilde"
|
||||
issue-term="title"
|
||||
crossorigin="anonymous"
|
||||
theme="github-dark"
|
||||
async>
|
||||
</script>
|
||||
|
||||
</div>
|
||||
<br>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
november 13 post mortem
|
||||
|
||||
we had something of an outage on november 13, 2018 on tilde.team.
|
||||
|
||||
i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.
|
||||
|
||||
tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.
|
||||
|
||||
> We have indications that there was an attack from your server.
|
||||
> Please take all necessary measures to avoid this in the future and to solve the issue.
|
||||
|
||||
at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.
|
||||
|
||||
when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the [webmail](https://mail.tilde.team) to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!
|
||||
|
||||
here, i launch in to full debugging mode: what command was it? who ran it?
|
||||
|
||||
search `~/.bash_history` per user was not very successful. nothing i could find was related to net or map. i had checked `sudo grep nmap /home/*/.bash_history` and many other commands.
|
||||
|
||||
at this point, i had connected with other ~teammates across other irc nets ([#!](https://hashbang.sh/), [~town](https://tilde.town), etc). among suggestions to check `/var/log/syslog`, `/var/log/kern.log`, and `dmesg`, i finally decided to check `ps`. `ps -ef | grep nmap` yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for [~fosslinux](/~fosslinux/).
|
||||
|
||||
i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police `nmap` when it isn't scanning on every port?
|
||||
|
||||
after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that [~fosslinux](/~fosslinux/) had only run `nmap` for addresses in the `10.0.0.0/8` space. the `10/8` address space is intended to not be addressable outside the local space. how could [hetzner](https://hetzner.com) have found out about a localhost network probe!?
|
||||
|
||||
finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.
|
||||
|
||||
it's definitely time to research redundancy options!
|
||||
|
||||
|
||||
tags: post-mortem, linux, sysadmin
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
|
||||
<meta name="theme-color" content="#00cc00">
|
||||
<link rel="icon" type="image/png" sizes="192x192" href="https://tilde.team/apple-touch-icon-precomposed.png">
|
||||
<link rel="icon" type="image/png" sizes="96x96" href="https://tilde.team/favicon-96x96.png">
|
||||
|
||||
<link rel="stylesheet" href="https://tilde.team/css/hacker.css">
|
||||
<link rel="stylesheet" href="extra.css">
|
||||
|
||||
<link rel="alternate" type="application/rss+xml" title="subscribe to this page..." href="feed.rss" />
|
||||
<title>proactive redundancy</title>
|
||||
</head><body>
|
||||
<div class="container">
|
||||
|
||||
<div id="divbodyholder">
|
||||
<div class="headerholder"><div class="header">
|
||||
<div id="title">
|
||||
<h1 class="nomargin"><a class="ablack" href="https://tilde.team/~ben/blog/index.html">blog // ~ben</a></h1>
|
||||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<!-- entry begin -->
|
||||
<h3><a class="ablack" href="proactive-redundancy.html">
|
||||
proactive redundancy
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811151839.26# -->
|
||||
<div class="subtitle">November 15, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>after the <a href="november-13-post-mortem.html">fiasco</a> earlier this week, i've been taking steps to minimize
|
||||
the impact if tilde.team were to go down. it's still a large spof (single-point-of-failure), but i'm reasonably certain that at least the irc net will remain up and functional in the event of another outage. </p>
|
||||
|
||||
<p>the first thing that i set up was a handful of additional ircd nodes: see <a href="https://tilde.chat/wiki/?page=servers">the tilde.chat wiki</a> for a full list. slash.tilde.chat is on my personal vps, and bsd.tilde.chat is hosted on the bsd vps that i set up for tilde.team. </p>
|
||||
|
||||
<p>i added the ipv4 addresses for these machines, along with the ip for yourtilde.com as A records for tilde.chat, creating a dns round-robin. <code>host tilde.chat</code> will return all four. requesting the dns record will return any one of them, rotating them in a semi-random fashion. this means that when connecting to tilde.chat on 6697 for irc, you might end up on any of <code>{your,team,bsd,slash}.tilde.chat</code>. </p>
|
||||
|
||||
<p>this creates the additional problem that visiting the <a href="https://tilde.chat">tilde.chat site</a> will end up at any of those 4 machines in much the same way. for the moment, the site is deployed on all of the boxes, making site setup issues hard to <a href="https://tildegit.org/tildeverse/tilde.chat/issues/8">debug</a>. the solution to this problem is to use a subdomain as the roundrobin host, as other networks like freenode do (see <code>host chat.freenode.net</code> for the list of servers).</p>
|
||||
|
||||
<p>i'm not sure how to make any of the other services more resilient. it's something that i have been and will continue to research moving forward.</p>
|
||||
|
||||
<p>the other main step that i have taken to prevent the same issue from happening again was to configure the firewall to drop outgoing requests to the subnets as defined in <a href="https://tools.ietf.org/html/rfc1918">rfc 1918</a>.</p>
|
||||
|
||||
<p>i'd like to consider at least this risk to be mitigated.</p>
|
||||
|
||||
<p>thanks for reading,</p>
|
||||
|
||||
<p>~ben</p>
|
||||
|
||||
<p>tags: <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
<!-- text end -->
|
||||
<!-- entry end -->
|
||||
</div>
|
||||
<div id="footer">CC by-nc-nd <a href="https://tilde.team/~ben/">~ben</a> — <a href="mailto:ben@tilde.team">ben@tilde.team</a><br/>
|
||||
generated with <a href="https://tildegit.org/team/bashblog">bashblog</a>, a single bash script to easily create blogs like this one</div>
|
||||
</div></div>
|
||||
<script src="https://utteranc.es/client.js"
|
||||
repo="benharri/tilde"
|
||||
issue-term="title"
|
||||
crossorigin="anonymous"
|
||||
theme="github-dark"
|
||||
async>
|
||||
</script>
|
||||
|
||||
</div>
|
||||
<br>
|
||||
</body></html>
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
proactive redundancy
|
||||
|
||||
after the [fiasco](november-13-post-mortem.html) earlier this week, i've been taking steps to minimize
|
||||
the impact if tilde.team were to go down. it's still a large spof (single-point-of-failure), but i'm reasonably certain that at least the irc net will remain up and functional in the event of another outage.
|
||||
|
||||
the first thing that i set up was a handful of additional ircd nodes: see [the tilde.chat wiki](https://tilde.chat/wiki/?page=servers) for a full list. slash.tilde.chat is on my personal vps, and bsd.tilde.chat is hosted on the bsd vps that i set up for tilde.team.
|
||||
|
||||
i added the ipv4 addresses for these machines, along with the ip for yourtilde.com as A records for tilde.chat, creating a dns round-robin. `host tilde.chat` will return all four. requesting the dns record will return any one of them, rotating them in a semi-random fashion. this means that when connecting to tilde.chat on 6697 for irc, you might end up on any of `{your,team,bsd,slash}.tilde.chat`.
|
||||
|
||||
this creates the additional problem that visiting the [tilde.chat site](https://tilde.chat) will end up at any of those 4 machines in much the same way. for the moment, the site is deployed on all of the boxes, making site setup issues hard to [debug](https://tildegit.org/tildeverse/tilde.chat/issues/8). the solution to this problem is to use a subdomain as the roundrobin host, as other networks like freenode do (see `host chat.freenode.net` for the list of servers).
|
||||
|
||||
i'm not sure how to make any of the other services more resilient. it's something that i have been and will continue to research moving forward.
|
||||
|
||||
the other main step that i have taken to prevent the same issue from happening again was to configure the firewall to drop outgoing requests to the subnets as defined in [rfc 1918](https://tools.ietf.org/html/rfc1918).
|
||||
|
||||
i'd like to consider at least this risk to be mitigated.
|
||||
|
||||
thanks for reading,
|
||||
|
||||
~ben
|
||||
|
||||
|
||||
tags: sysadmin, tilde
|
||||
|
|
@ -24,41 +24,45 @@
|
|||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3><a class="ablack" href="no-more-google.html">
|
||||
no more google
|
||||
<h3><a class="ablack" href="november-13-post-mortem.html">
|
||||
november 13 post mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201808142336.05# -->
|
||||
<div class="subtitle">August 14, 2018 —
|
||||
<!-- bashblog_timestamp: #201811132020.33# -->
|
||||
<div class="subtitle">November 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>not sure if this is appropriately tagged, but i didn't feel like making a new
|
||||
one.</p>
|
||||
<p>we had something of an outage on november 13, 2018 on tilde.team. </p>
|
||||
|
||||
<p>i figured i should probably get some notes down about moving off google.</p>
|
||||
<p>i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.</p>
|
||||
|
||||
<p>to start, i'll get a list of the things i was able to easily replace:</p>
|
||||
<p>tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.</p>
|
||||
|
||||
<ul>
|
||||
<li>gmail => <a href="https://tilde.team/wiki/?page=email">@tilde.team mail</a></li>
|
||||
<li>google drive => <a href="https://syncthing.net">syncthing</a> (with a persistent node running on my personal vps)</li>
|
||||
</ul>
|
||||
<blockquote>
|
||||
<p>We have indications that there was an attack from your server.
|
||||
Please take all necessary measures to avoid this in the future and to solve the issue.</p>
|
||||
</blockquote>
|
||||
|
||||
<p>i'm still using:</p>
|
||||
<p>at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.</p>
|
||||
|
||||
<ul>
|
||||
<li>gplay music/youtube</li>
|
||||
<li>google maps (open streetmap isn't good enough to replace it)</li>
|
||||
<li>google photos - but this is going to be replaced long-term with syncthing</li>
|
||||
</ul>
|
||||
<p>when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the <a href="https://mail.tilde.team">webmail</a> to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_net-neutrality.html'>net-neutrality</a></p>
|
||||
<p>here, i launch in to full debugging mode: what command was it? who ran it? </p>
|
||||
|
||||
<p>search <code>~/.bash_history</code> per user was not very successful. nothing i could find was related to net or map. i had checked <code>sudo grep nmap /home/*/.bash_history</code> and many other commands. </p>
|
||||
|
||||
<p>at this point, i had connected with other ~teammates across other irc nets (<a href="https://hashbang.sh/">#!</a>, <a href="https://tilde.town">~town</a>, etc). among suggestions to check <code>/var/log/syslog</code>, <code>/var/log/kern.log</code>, and <code>dmesg</code>, i finally decided to check <code>ps</code>. <code>ps -ef | grep nmap</code> yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for <a href="/~fosslinux/">~fosslinux</a>.</p>
|
||||
|
||||
<p>i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police <code>nmap</code> when it isn't scanning on every port?</p>
|
||||
|
||||
<p>after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that <a href="/~fosslinux/">~fosslinux</a> had only run <code>nmap</code> for addresses in the <code>10.0.0.0/8</code> space. the <code>10/8</code> address space is intended to not be addressable outside the local space. how could <a href="https://hetzner.com">hetzner</a> have found out about a localhost network probe!?</p>
|
||||
|
||||
<p>finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.</p>
|
||||
|
||||
<p>it's definitely time to research redundancy options!</p>
|
||||
|
||||
<p>tags: <a href='tag_post-mortem.html'>post-mortem</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="upsides-of-new-dns-nameservers.html">
|
||||
upsides of new dns nameservers
|
||||
|
|
@ -129,35 +133,6 @@ as soon as it all propagates through the fickle beast that is dns.</p>
|
|||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="lxd-networking-and-additional-ips.html">
|
||||
lxd networking and additional IPs
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807261534.50# -->
|
||||
<div class="subtitle">July 26, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>now that tilde.team is on a fancy-shmancy new dedi server, i've tried to get a secondary IP address
|
||||
assigned to a lxd container (which i plan to use for my personal stuff). lxd shows that the secondary
|
||||
IP is being picked up by that container, but i'm still seeing the host machine's IP as the external
|
||||
address.</p>
|
||||
|
||||
<p>i'm not sure how i'll need to configure the network settings on the host machine (now that we're running
|
||||
ubuntu 18.04 and it uses netplan for configs and not /etc/network/interfaces). another confusing thing is
|
||||
that the main config in /etc/netplan says that the network config is handled by systemd-networkd...</p>
|
||||
|
||||
<p>at least i have through the end of the year when my current vps runs out to get this up and running.</p>
|
||||
|
||||
<p>ping me on <a href="https://tilde.chat">irc</a> or <a href="mailto:ben@tilde.team">email</a> if you have experience with this.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_ubuntu.html'>ubuntu</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="dotfiles.html">
|
||||
dotfiles
|
||||
|
|
@ -231,6 +206,71 @@ tildeman
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="lxd-networking-and-additional-ips.html">
|
||||
lxd networking and additional IPs
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807261534.50# -->
|
||||
<div class="subtitle">July 26, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>now that tilde.team is on a fancy-shmancy new dedi server, i've tried to get a secondary IP address
|
||||
assigned to a lxd container (which i plan to use for my personal stuff). lxd shows that the secondary
|
||||
IP is being picked up by that container, but i'm still seeing the host machine's IP as the external
|
||||
address.</p>
|
||||
|
||||
<p>i'm not sure how i'll need to configure the network settings on the host machine (now that we're running
|
||||
ubuntu 18.04 and it uses netplan for configs and not /etc/network/interfaces). another confusing thing is
|
||||
that the main config in /etc/netplan says that the network config is handled by systemd-networkd...</p>
|
||||
|
||||
<p>at least i have through the end of the year when my current vps runs out to get this up and running.</p>
|
||||
|
||||
<p>ping me on <a href="https://tilde.chat">irc</a> or <a href="mailto:ben@tilde.team">email</a> if you have experience with this.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_ubuntu.html'>ubuntu</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="no-more-google.html">
|
||||
no more google
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201808142336.05# -->
|
||||
<div class="subtitle">August 14, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>not sure if this is appropriately tagged, but i didn't feel like making a new
|
||||
one.</p>
|
||||
|
||||
<p>i figured i should probably get some notes down about moving off google.</p>
|
||||
|
||||
<p>to start, i'll get a list of the things i was able to easily replace:</p>
|
||||
|
||||
<ul>
|
||||
<li>gmail => <a href="https://tilde.team/wiki/?page=email">@tilde.team mail</a></li>
|
||||
<li>google drive => <a href="https://syncthing.net">syncthing</a> (with a persistent node running on my personal vps)</li>
|
||||
</ul>
|
||||
|
||||
<p>i'm still using:</p>
|
||||
|
||||
<ul>
|
||||
<li>gplay music/youtube</li>
|
||||
<li>google maps (open streetmap isn't good enough to replace it)</li>
|
||||
<li>google photos - but this is going to be replaced long-term with syncthing</li>
|
||||
</ul>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_net-neutrality.html'>net-neutrality</a></p>
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,81 @@
|
|||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
|
||||
|
||||
<meta name="theme-color" content="#00cc00">
|
||||
<link rel="icon" type="image/png" sizes="192x192" href="https://tilde.team/apple-touch-icon-precomposed.png">
|
||||
<link rel="icon" type="image/png" sizes="96x96" href="https://tilde.team/favicon-96x96.png">
|
||||
|
||||
<link rel="stylesheet" href="https://tilde.team/css/hacker.css">
|
||||
<link rel="stylesheet" href="extra.css">
|
||||
|
||||
<link rel="alternate" type="application/rss+xml" title="subscribe to this page..." href="feed.rss" />
|
||||
<title>blog // ~ben — posts tagged "post-mortem"</title>
|
||||
</head><body>
|
||||
<div class="container">
|
||||
|
||||
<div id="divbodyholder">
|
||||
<div class="headerholder"><div class="header">
|
||||
<div id="title">
|
||||
<h1 class="nomargin"><a class="ablack" href="https://tilde.team/~ben/blog/index.html">blog // ~ben</a></h1>
|
||||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3><a class="ablack" href="november-13-post-mortem.html">
|
||||
november 13 post mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811132020.33# -->
|
||||
<div class="subtitle">November 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>we had something of an outage on november 13, 2018 on tilde.team. </p>
|
||||
|
||||
<p>i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.</p>
|
||||
|
||||
<p>tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.</p>
|
||||
|
||||
<blockquote>
|
||||
<p>We have indications that there was an attack from your server.
|
||||
Please take all necessary measures to avoid this in the future and to solve the issue.</p>
|
||||
</blockquote>
|
||||
|
||||
<p>at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.</p>
|
||||
|
||||
<p>when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the <a href="https://mail.tilde.team">webmail</a> to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!</p>
|
||||
|
||||
<p>here, i launch in to full debugging mode: what command was it? who ran it? </p>
|
||||
|
||||
<p>search <code>~/.bash_history</code> per user was not very successful. nothing i could find was related to net or map. i had checked <code>sudo grep nmap /home/*/.bash_history</code> and many other commands. </p>
|
||||
|
||||
<p>at this point, i had connected with other ~teammates across other irc nets (<a href="https://hashbang.sh/">#!</a>, <a href="https://tilde.town">~town</a>, etc). among suggestions to check <code>/var/log/syslog</code>, <code>/var/log/kern.log</code>, and <code>dmesg</code>, i finally decided to check <code>ps</code>. <code>ps -ef | grep nmap</code> yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for <a href="/~fosslinux/">~fosslinux</a>.</p>
|
||||
|
||||
<p>i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police <code>nmap</code> when it isn't scanning on every port?</p>
|
||||
|
||||
<p>after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that <a href="/~fosslinux/">~fosslinux</a> had only run <code>nmap</code> for addresses in the <code>10.0.0.0/8</code> space. the <code>10/8</code> address space is intended to not be addressable outside the local space. how could <a href="https://hetzner.com">hetzner</a> have found out about a localhost network probe!?</p>
|
||||
|
||||
<p>finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.</p>
|
||||
|
||||
<p>it's definitely time to research redundancy options!</p>
|
||||
|
||||
<p>tags: <a href='tag_post-mortem.html'>post-mortem</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
<!-- text end -->
|
||||
</div>
|
||||
<div id="footer">CC by-nc-nd <a href="https://tilde.team/~ben/">~ben</a> — <a href="mailto:ben@tilde.team">ben@tilde.team</a><br/>
|
||||
generated with <a href="https://tildegit.org/team/bashblog">bashblog</a>, a single bash script to easily create blogs like this one</div>
|
||||
</div></div>
|
||||
<script src="https://utteranc.es/client.js"
|
||||
repo="benharri/tilde"
|
||||
issue-term="title"
|
||||
crossorigin="anonymous"
|
||||
theme="github-dark"
|
||||
async>
|
||||
</script>
|
||||
|
||||
</div>
|
||||
<br>
|
||||
</body></html>
|
||||
|
|
@ -24,6 +24,113 @@
|
|||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3><a class="ablack" href="proactive-redundancy.html">
|
||||
proactive redundancy
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811151839.26# -->
|
||||
<div class="subtitle">November 15, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>after the <a href="november-13-post-mortem.html">fiasco</a> earlier this week, i've been taking steps to minimize
|
||||
the impact if tilde.team were to go down. it's still a large spof (single-point-of-failure), but i'm reasonably certain that at least the irc net will remain up and functional in the event of another outage. </p>
|
||||
|
||||
<p>the first thing that i set up was a handful of additional ircd nodes: see <a href="https://tilde.chat/wiki/?page=servers">the tilde.chat wiki</a> for a full list. slash.tilde.chat is on my personal vps, and bsd.tilde.chat is hosted on the bsd vps that i set up for tilde.team. </p>
|
||||
|
||||
<p>i added the ipv4 addresses for these machines, along with the ip for yourtilde.com as A records for tilde.chat, creating a dns round-robin. <code>host tilde.chat</code> will return all four. requesting the dns record will return any one of them, rotating them in a semi-random fashion. this means that when connecting to tilde.chat on 6697 for irc, you might end up on any of <code>{your,team,bsd,slash}.tilde.chat</code>. </p>
|
||||
|
||||
<p>this creates the additional problem that visiting the <a href="https://tilde.chat">tilde.chat site</a> will end up at any of those 4 machines in much the same way. for the moment, the site is deployed on all of the boxes, making site setup issues hard to <a href="https://tildegit.org/tildeverse/tilde.chat/issues/8">debug</a>. the solution to this problem is to use a subdomain as the roundrobin host, as other networks like freenode do (see <code>host chat.freenode.net</code> for the list of servers).</p>
|
||||
|
||||
<p>i'm not sure how to make any of the other services more resilient. it's something that i have been and will continue to research moving forward.</p>
|
||||
|
||||
<p>the other main step that i have taken to prevent the same issue from happening again was to configure the firewall to drop outgoing requests to the subnets as defined in <a href="https://tools.ietf.org/html/rfc1918">rfc 1918</a>.</p>
|
||||
|
||||
<p>i'd like to consider at least this risk to be mitigated.</p>
|
||||
|
||||
<p>thanks for reading,</p>
|
||||
|
||||
<p>~ben</p>
|
||||
|
||||
<p>tags: <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="november-13-post-mortem.html">
|
||||
november 13 post mortem
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811132020.33# -->
|
||||
<div class="subtitle">November 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>we had something of an outage on november 13, 2018 on tilde.team. </p>
|
||||
|
||||
<p>i awoke, not suspecting anything to be amiss. as soon as i logged in to check my email and irc mentions, it became clear.</p>
|
||||
|
||||
<p>tilde.team was at the least inaccessible, and at the worst, down completely. according the message in my inbox, there hade been an attempted "attack" from my IP.</p>
|
||||
|
||||
<blockquote>
|
||||
<p>We have indications that there was an attack from your server.
|
||||
Please take all necessary measures to avoid this in the future and to solve the issue.</p>
|
||||
</blockquote>
|
||||
|
||||
<p>at this point, i have no idea what could have happened over night while i'm sleeping. the timestamp shows that it arrive only 30 minutes after i'd turned in for the night.</p>
|
||||
|
||||
<p>when i finally log on in the morning to check mails and irc mentions, i find that i'm unable to connect to tilde.team... strange, but ok; time to troubleshoot. i refresh the <a href="https://mail.tilde.team">webmail</a> to see what i'm missing. it ends up failing to find the server. even stranger! i'd better get the mails off my phone if they're on my @tilde.team mail!</p>
|
||||
|
||||
<p>here, i launch in to full debugging mode: what command was it? who ran it? </p>
|
||||
|
||||
<p>search <code>~/.bash_history</code> per user was not very successful. nothing i could find was related to net or map. i had checked <code>sudo grep nmap /home/*/.bash_history</code> and many other commands. </p>
|
||||
|
||||
<p>at this point, i had connected with other ~teammates across other irc nets (<a href="https://hashbang.sh/">#!</a>, <a href="https://tilde.town">~town</a>, etc). among suggestions to check <code>/var/log/syslog</code>, <code>/var/log/kern.log</code>, and <code>dmesg</code>, i finally decided to check <code>ps</code>. <code>ps -ef | grep nmap</code> yielded nmap on an obscured uid and gid, which is shortly established to belong to a container i had provisioned for <a href="/~fosslinux/">~fosslinux</a>.</p>
|
||||
|
||||
<p>i'm not considering methods of policing access to any site over port 80 and port 443. this is crazy. how do you police <code>nmap</code> when it isn't scanning on every port?</p>
|
||||
|
||||
<p>after a bit of shit-talking and reassurance from other sysadmins, i reexamined and realized that <a href="/~fosslinux/">~fosslinux</a> had only run <code>nmap</code> for addresses in the <code>10.0.0.0/8</code> space. the <code>10/8</code> address space is intended to not be addressable outside the local space. how could <a href="https://hetzner.com">hetzner</a> have found out about a localhost network probe!?</p>
|
||||
|
||||
<p>finally, after speaking with more people than i expected to speak with in one day, i ended up sending three different support emails to hetzner support, which finally resulted in them unlocking the ip.</p>
|
||||
|
||||
<p>it's definitely time to research redundancy options!</p>
|
||||
|
||||
<p>tags: <a href='tag_post-mortem.html'>post-mortem</a>, <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="tildeteam-news.html">
|
||||
tilde.team news
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201806131507.45# -->
|
||||
<div class="subtitle">June 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hey hi hello!</p>
|
||||
|
||||
<p>it seems that i haven't written anything on my blog in quite a while...</p>
|
||||
|
||||
<p>time to fix that! i've been quite busy in the last month or so with a lot of new ideas an energy for tilde.team.</p>
|
||||
|
||||
<p>after rediscovering my account on tilde.town, i hopped in the irc there and my enthusiasm translated into a couple new members over here on the ~team.</p>
|
||||
|
||||
<p>our irc has been somewhat more active recently which is awesome:)</p>
|
||||
|
||||
<p>some of the new updates in the last month:</p>
|
||||
|
||||
<ul>
|
||||
<li><a href="https://git.tilde.team">tildegit (our own gitea instance)</a></li>
|
||||
<li><a href="https://mail.tilde.team">tildemail</a> with postfix and dovecot for smtp/imap as well as local command line mail in mutt and alpine</li>
|
||||
<li><a href="https://git.tildeverse.org/team/tilde-launcher"><code>tilde</code></a> user script wrapper with submission and approval flows</li>
|
||||
<li><a href="https://tilde.team/wiki/?page=ssh">password auth disabled</a></li>
|
||||
</ul>
|
||||
|
||||
<p>i'd like to make use of our new mailserver, so shoot me some <a href="mailto:ben@tilde.team">mail</a>.
|
||||
i never get enough personal mail. it's all still privacy policy update notices. :(</p>
|
||||
|
||||
<p>see you soon!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="dns-shenanigans-post-mortem.html">
|
||||
dns shenanigans post-mortem
|
||||
</a></h3>
|
||||
|
|
@ -71,72 +178,6 @@ as soon as it all propagates through the fickle beast that is dns.</p>
|
|||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="lxd-networking-and-additional-ips.html">
|
||||
lxd networking and additional IPs
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807261534.50# -->
|
||||
<div class="subtitle">July 26, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>now that tilde.team is on a fancy-shmancy new dedi server, i've tried to get a secondary IP address
|
||||
assigned to a lxd container (which i plan to use for my personal stuff). lxd shows that the secondary
|
||||
IP is being picked up by that container, but i'm still seeing the host machine's IP as the external
|
||||
address.</p>
|
||||
|
||||
<p>i'm not sure how i'll need to configure the network settings on the host machine (now that we're running
|
||||
ubuntu 18.04 and it uses netplan for configs and not /etc/network/interfaces). another confusing thing is
|
||||
that the main config in /etc/netplan says that the network config is handled by systemd-networkd...</p>
|
||||
|
||||
<p>at least i have through the end of the year when my current vps runs out to get this up and running.</p>
|
||||
|
||||
<p>ping me on <a href="https://tilde.chat">irc</a> or <a href="mailto:ben@tilde.team">email</a> if you have experience with this.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_ubuntu.html'>ubuntu</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="tildeteam-news.html">
|
||||
tilde.team news
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201806131507.45# -->
|
||||
<div class="subtitle">June 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hey hi hello!</p>
|
||||
|
||||
<p>it seems that i haven't written anything on my blog in quite a while...</p>
|
||||
|
||||
<p>time to fix that! i've been quite busy in the last month or so with a lot of new ideas an energy for tilde.team.</p>
|
||||
|
||||
<p>after rediscovering my account on tilde.town, i hopped in the irc there and my enthusiasm translated into a couple new members over here on the ~team.</p>
|
||||
|
||||
<p>our irc has been somewhat more active recently which is awesome:)</p>
|
||||
|
||||
<p>some of the new updates in the last month:</p>
|
||||
|
||||
<ul>
|
||||
<li><a href="https://git.tilde.team">tildegit (our own gitea instance)</a></li>
|
||||
<li><a href="https://mail.tilde.team">tildemail</a> with postfix and dovecot for smtp/imap as well as local command line mail in mutt and alpine</li>
|
||||
<li><a href="https://git.tildeverse.org/team/tilde-launcher"><code>tilde</code></a> user script wrapper with submission and approval flows</li>
|
||||
<li><a href="https://tilde.team/wiki/?page=ssh">password auth disabled</a></li>
|
||||
</ul>
|
||||
|
||||
<p>i'd like to make use of our new mailserver, so shoot me some <a href="mailto:ben@tilde.team">mail</a>.
|
||||
i never get enough personal mail. it's all still privacy policy update notices. :(</p>
|
||||
|
||||
<p>see you soon!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="hi-there.html">
|
||||
hi there
|
||||
|
|
@ -181,6 +222,35 @@ tildeman
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="lxd-networking-and-additional-ips.html">
|
||||
lxd networking and additional IPs
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807261534.50# -->
|
||||
<div class="subtitle">July 26, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>now that tilde.team is on a fancy-shmancy new dedi server, i've tried to get a secondary IP address
|
||||
assigned to a lxd container (which i plan to use for my personal stuff). lxd shows that the secondary
|
||||
IP is being picked up by that container, but i'm still seeing the host machine's IP as the external
|
||||
address.</p>
|
||||
|
||||
<p>i'm not sure how i'll need to configure the network settings on the host machine (now that we're running
|
||||
ubuntu 18.04 and it uses netplan for configs and not /etc/network/interfaces). another confusing thing is
|
||||
that the main config in /etc/netplan says that the network config is handled by systemd-networkd...</p>
|
||||
|
||||
<p>at least i have through the end of the year when my current vps runs out to get this up and running.</p>
|
||||
|
||||
<p>ping me on <a href="https://tilde.chat">irc</a> or <a href="mailto:ben@tilde.team">email</a> if you have experience with this.</p>
|
||||
|
||||
<p>tags: <a href='tag_linux.html'>linux</a>, <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_ubuntu.html'>ubuntu</a></p>
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,73 @@
|
|||
<div id="description">a blog about tildes and other things</div>
|
||||
</div></div></div>
|
||||
<div id="divbody"><div class="content">
|
||||
<h3><a class="ablack" href="proactive-redundancy.html">
|
||||
proactive redundancy
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201811151839.26# -->
|
||||
<div class="subtitle">November 15, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>after the <a href="november-13-post-mortem.html">fiasco</a> earlier this week, i've been taking steps to minimize
|
||||
the impact if tilde.team were to go down. it's still a large spof (single-point-of-failure), but i'm reasonably certain that at least the irc net will remain up and functional in the event of another outage. </p>
|
||||
|
||||
<p>the first thing that i set up was a handful of additional ircd nodes: see <a href="https://tilde.chat/wiki/?page=servers">the tilde.chat wiki</a> for a full list. slash.tilde.chat is on my personal vps, and bsd.tilde.chat is hosted on the bsd vps that i set up for tilde.team. </p>
|
||||
|
||||
<p>i added the ipv4 addresses for these machines, along with the ip for yourtilde.com as A records for tilde.chat, creating a dns round-robin. <code>host tilde.chat</code> will return all four. requesting the dns record will return any one of them, rotating them in a semi-random fashion. this means that when connecting to tilde.chat on 6697 for irc, you might end up on any of <code>{your,team,bsd,slash}.tilde.chat</code>. </p>
|
||||
|
||||
<p>this creates the additional problem that visiting the <a href="https://tilde.chat">tilde.chat site</a> will end up at any of those 4 machines in much the same way. for the moment, the site is deployed on all of the boxes, making site setup issues hard to <a href="https://tildegit.org/tildeverse/tilde.chat/issues/8">debug</a>. the solution to this problem is to use a subdomain as the roundrobin host, as other networks like freenode do (see <code>host chat.freenode.net</code> for the list of servers).</p>
|
||||
|
||||
<p>i'm not sure how to make any of the other services more resilient. it's something that i have been and will continue to research moving forward.</p>
|
||||
|
||||
<p>the other main step that i have taken to prevent the same issue from happening again was to configure the firewall to drop outgoing requests to the subnets as defined in <a href="https://tools.ietf.org/html/rfc1918">rfc 1918</a>.</p>
|
||||
|
||||
<p>i'd like to consider at least this risk to be mitigated.</p>
|
||||
|
||||
<p>thanks for reading,</p>
|
||||
|
||||
<p>~ben</p>
|
||||
|
||||
<p>tags: <a href='tag_sysadmin.html'>sysadmin</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="tildeteam-news.html">
|
||||
tilde.team news
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201806131507.45# -->
|
||||
<div class="subtitle">June 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hey hi hello!</p>
|
||||
|
||||
<p>it seems that i haven't written anything on my blog in quite a while...</p>
|
||||
|
||||
<p>time to fix that! i've been quite busy in the last month or so with a lot of new ideas an energy for tilde.team.</p>
|
||||
|
||||
<p>after rediscovering my account on tilde.town, i hopped in the irc there and my enthusiasm translated into a couple new members over here on the ~team.</p>
|
||||
|
||||
<p>our irc has been somewhat more active recently which is awesome:)</p>
|
||||
|
||||
<p>some of the new updates in the last month:</p>
|
||||
|
||||
<ul>
|
||||
<li><a href="https://git.tilde.team">tildegit (our own gitea instance)</a></li>
|
||||
<li><a href="https://mail.tilde.team">tildemail</a> with postfix and dovecot for smtp/imap as well as local command line mail in mutt and alpine</li>
|
||||
<li><a href="https://git.tildeverse.org/team/tilde-launcher"><code>tilde</code></a> user script wrapper with submission and approval flows</li>
|
||||
<li><a href="https://tilde.team/wiki/?page=ssh">password auth disabled</a></li>
|
||||
</ul>
|
||||
|
||||
<p>i'd like to make use of our new mailserver, so shoot me some <a href="mailto:ben@tilde.team">mail</a>.
|
||||
i never get enough personal mail. it's all still privacy policy update notices. :(</p>
|
||||
|
||||
<p>see you soon!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="upsides-of-new-dns-nameservers.html">
|
||||
upsides of new dns nameservers
|
||||
</a></h3>
|
||||
|
|
@ -45,6 +112,34 @@ upsides of new dns nameservers
|
|||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="bashblog-and-your-gopherhole.html">
|
||||
bashblog and your gopherhole
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807221144.03# -->
|
||||
<div class="subtitle">July 22, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i've created <a href="https://git.tildeverse.org/meta/bashblog">a repo</a> for the tilde.team customizations to <a href="https://github.com/cfenollosa/bashblog">bashblog</a>.</p>
|
||||
|
||||
<p>it will now make sure that your ~/public_gopher exists and symlink your blog into it with a nice gophermap to list all the markdown styled posts.</p>
|
||||
|
||||
<p>try it out and let me know if there are any problems!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_blog.html'>blog</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="dns-shenanigans-post-mortem.html">
|
||||
dns shenanigans post-mortem
|
||||
|
|
@ -93,109 +188,6 @@ as soon as it all propagates through the fickle beast that is dns.</p>
|
|||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="bashblog-and-your-gopherhole.html">
|
||||
bashblog and your gopherhole
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201807221144.03# -->
|
||||
<div class="subtitle">July 22, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>i've created <a href="https://git.tildeverse.org/meta/bashblog">a repo</a> for the tilde.team customizations to <a href="https://github.com/cfenollosa/bashblog">bashblog</a>.</p>
|
||||
|
||||
<p>it will now make sure that your ~/public_gopher exists and symlink your blog into it with a nice gophermap to list all the markdown styled posts.</p>
|
||||
|
||||
<p>try it out and let me know if there are any problems!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_blog.html'>blog</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="tildeteam-news.html">
|
||||
tilde.team news
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201806131507.45# -->
|
||||
<div class="subtitle">June 13, 2018 —
|
||||
~ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hey hi hello!</p>
|
||||
|
||||
<p>it seems that i haven't written anything on my blog in quite a while...</p>
|
||||
|
||||
<p>time to fix that! i've been quite busy in the last month or so with a lot of new ideas an energy for tilde.team.</p>
|
||||
|
||||
<p>after rediscovering my account on tilde.town, i hopped in the irc there and my enthusiasm translated into a couple new members over here on the ~team.</p>
|
||||
|
||||
<p>our irc has been somewhat more active recently which is awesome:)</p>
|
||||
|
||||
<p>some of the new updates in the last month:</p>
|
||||
|
||||
<ul>
|
||||
<li><a href="https://git.tilde.team">tildegit (our own gitea instance)</a></li>
|
||||
<li><a href="https://mail.tilde.team">tildemail</a> with postfix and dovecot for smtp/imap as well as local command line mail in mutt and alpine</li>
|
||||
<li><a href="https://git.tildeverse.org/team/tilde-launcher"><code>tilde</code></a> user script wrapper with submission and approval flows</li>
|
||||
<li><a href="https://tilde.team/wiki/?page=ssh">password auth disabled</a></li>
|
||||
</ul>
|
||||
|
||||
<p>i'd like to make use of our new mailserver, so shoot me some <a href="mailto:ben@tilde.team">mail</a>.
|
||||
i never get enough personal mail. it's all still privacy policy update notices. :(</p>
|
||||
|
||||
<p>see you soon!</p>
|
||||
|
||||
<p>tags: <a href='tag_tilde.html'>tilde</a>, <a href='tag_sysadmin.html'>sysadmin</a></p>
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="mastodon.html">
|
||||
mastodon
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201712221628.45# -->
|
||||
<div class="subtitle">December 22, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hi everyone.</p>
|
||||
|
||||
<p>i started hosting a <a href="https://joinmastodon.org">mastodon</a> instance at <a href="https://social.tilde.team">social.tilde.team</a>.</p>
|
||||
|
||||
<p>check it out if you want some federated open source social goodness :)</p>
|
||||
|
||||
<p>send me a toot <a href="https://social.tilde.team/@ben">@ben@tilde.team</a> (from any mastodon instance!)</p>
|
||||
|
||||
<p>thanks!</p>
|
||||
|
||||
<p>tags: <a href='tag_social-networks.html'>social-networks</a>, <a href='tag_mastodon.html'>mastodon</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="hi-there.html">
|
||||
hi there
|
||||
|
|
@ -236,6 +228,44 @@ tildeman
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- text end -->
|
||||
<h3><a class="ablack" href="mastodon.html">
|
||||
mastodon
|
||||
</a></h3>
|
||||
<!-- bashblog_timestamp: #201712221628.45# -->
|
||||
<div class="subtitle">December 22, 2017 —
|
||||
ben
|
||||
</div>
|
||||
<!-- text begin -->
|
||||
|
||||
<p>hi everyone.</p>
|
||||
|
||||
<p>i started hosting a <a href="https://joinmastodon.org">mastodon</a> instance at <a href="https://social.tilde.team">social.tilde.team</a>.</p>
|
||||
|
||||
<p>check it out if you want some federated open source social goodness :)</p>
|
||||
|
||||
<p>send me a toot <a href="https://social.tilde.team/@ben">@ben@tilde.team</a> (from any mastodon instance!)</p>
|
||||
|
||||
<p>thanks!</p>
|
||||
|
||||
<p>tags: <a href='tag_social-networks.html'>social-networks</a>, <a href='tag_mastodon.html'>mastodon</a>, <a href='tag_tilde.html'>tilde</a></p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC0Sf4AJXVn1CKWLPTXoV+bi3LNTGrGDPjWNGnJsIC11frQxDH1D4MM72WBTTERjT2ZpV48sZaPrmeQj960n7RxQAwoE0sDBMO4aKs3TXtaAY+RJarVSRPP3beo6ELONYRvdymnNPOxInWAzsPPjunZ0ufQDUt6Iyy+hTb7FsNgq4A736CxMb4oGSP/AT2Yfb0jUX0vtCxvgQC2T0oYrj8OInv8Qu0/0I0YQErZlkUO92b47B39hMxAIeTX2+0M0g8M5nJAc05gK6eoq27amysXsSm8aM/5ikY1Z4mabfAFtXzxsdFNzQ2TYCqbEnXUr5tTLyjfGYkSd97pbJtaaDBC+NAfX5m8QmrgG/5Ad9dlQH6g5JLng7yeRJYfzTlVW2gvmMnrAplHjCEIwCO0LA4zotzs6XkVrxQDmj2VGDdyFu4Z8fv9Ruf0lCE8PG03TnX7yJiE9RtT6KX3m+12xr71rdfVONRuZVkQlPlGKllGLQ7ZM5QJ2aVPDI1s2ZHFF9bm8lzg8u49KEmAJZpZPm+wc1b43inuiN/ftcl7oe1ZIxilUmJXaG74xbhfLSZa+S93WryhZ9cjoSy9p3q0QmZZh2ePSNyc6R3xJQnPjDozpJ0D3CRVeGJomGHS2E8wrHtWs5L2q0A3CjlVaducD1M2MgnQNEDmiKVob1utOKwrLw== openpgp:0x2206A906
|
||||
Loading…
Reference in New Issue