mirrors/tt-rss
1
0
Fork 0
mirror of https://git.tt-rss.org/fox/tt-rss.git synced 2025-12-15 18:00:59 +01:00
Code Issues Projects Releases Wiki Activity

fork masterminds html5-php

Browse Source
This commit is contained in:
Andrew Dolgov 2022-07-31 09:15:00 +03:00
parent 4aefbd628e
commit 7187ab859d
No known key found for this signature in database
GPG Key ID: 1A56B4FA25D4AF2A
145 changed files with 59633 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
phpstan.neon
View File

@ -1,7 +1,7 @@
parameters:
level: 6
parallel:
maximumNumberOfProcesses: 4
maximumNumberOfProcesses: 2
reportUnmatchedIgnoredErrors: false
ignoreErrors:
- '#Constant.*\b(SUBSTRING_FOR_DATE|SCHEMA_VERSION|SELF_USER_AGENT|LABEL_BASE_INDEX|PLUGIN_FEED_BASE_INDEX)\b.*not found#'

13
plugins/af_readability/composer.json
View File

@ -1,9 +1,16 @@
{
"minimum-stability": "dev",
"prefer-stable": true,
"repositories": [
{
"name": "fivefilters/readability.php",
"type": "vcs",
"url": "https://dev.tt-rss.org/fox/readability-php.git"
"name": "fivefilters/readability.php",
"type": "vcs",
"url": "https://dev.tt-rss.org/fox/readability-php.git"
},
{
"name": "masterminds/html5",
"type": "vcs",
"url": "https://dev.tt-rss.org/fox/html5-php.git"
}
],
"require": {

35
plugins/af_readability/composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "351bb4bce6353ca338c03626bab6413f",
"content-hash": "183ed768c66eb8f183350edf06c06a63",
"packages": [
{
"name": "fivefilters/readability.php",
@ -12,14 +12,14 @@
"source": {
"type": "git",
"url": "https://dev.tt-rss.org/fox/readability-php.git",
"reference": "5ad152c70376002f043bb936d8ae5eed103fb993"
"reference": "8ac5abdd497b37d2be4833bcf18d6819bba4d9c9"
},
"require": {
"ext-dom": "*",
"ext-mbstring": "*",
"ext-xml": "*",
"league/uri": "^6.4",
"masterminds/html5": "^2.0",
"masterminds/html5": "2.7.x-dev@dev",
"php": ">=7.3.0",
"psr/log": "^1.0"
},
@ -64,7 +64,7 @@
"html",
"readability"
],
"time": "2022-07-16T13:23:08+00:00"
"time": "2022-07-31T06:02:47+00:00"
},
{
"name": "league/uri",
@ -238,18 +238,12 @@
},
{
"name": "masterminds/html5",
"version": "2.7.5",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/Masterminds/html5-php.git",
"url": "https://dev.tt-rss.org/fox/html5-php.git",
"reference": "f640ac1bdddff06ea333a920c95bbad8872429ab"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Masterminds/html5-php/zipball/f640ac1bdddff06ea333a920c95bbad8872429ab",
"reference": "f640ac1bdddff06ea333a920c95bbad8872429ab",
"shasum": ""
},
"require": {
"ext-ctype": "*",
"ext-dom": "*",
@ -259,6 +253,7 @@
"require-dev": {
"phpunit/phpunit": "^4.8.35 || ^5.7.21 || ^6 || ^7"
},
"default-branch": true,
"type": "library",
"extra": {
"branch-alias": {
@ -270,7 +265,11 @@
"Masterminds\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"autoload-dev": {
"psr-4": {
"Masterminds\\HTML5\\Tests\\": "test/HTML5"
}
},
"license": [
"MIT"
],
@ -291,18 +290,14 @@
"description": "An HTML5 parser and serializer.",
"homepage": "http://masterminds.github.io/html5-php",
"keywords": [
"HTML5",
"dom",
"html",
"html5",
"parser",
"querypath",
"serializer",
"xml"
],
"support": {
"issues": "https://github.com/Masterminds/html5-php/issues",
"source": "https://github.com/Masterminds/html5-php/tree/2.7.5"
},
"time": "2021-07-01T14:25:37+00:00"
},
{
@ -466,11 +461,11 @@
],
"packages-dev": [],
"aliases": [],
"minimum-stability": "stable",
"minimum-stability": "dev",
"stability-flags": {
"fivefilters/readability.php": 20
},
"prefer-stable": false,
"prefer-stable": true,
"prefer-lowest": false,
"platform": [],
"platform-dev": [],

11
plugins/af_readability/vendor/composer/InstalledVersions.php vendored
View File

@ -30,7 +30,7 @@ class InstalledVersions
'aliases' =>
array (
),
'reference' => '5006c754c42a09f5b88b258c2da4b9eff7508357',
'reference' => '4aefbd628e9a0e1eac58523904ad887b0635cda3',
'name' => '__root__',
),
'versions' =>
@ -42,7 +42,7 @@ class InstalledVersions
'aliases' =>
array (
),
'reference' => '5006c754c42a09f5b88b258c2da4b9eff7508357',
'reference' => '4aefbd628e9a0e1eac58523904ad887b0635cda3',
),
'fivefilters/readability.php' =>
array (
@ -52,7 +52,7 @@ class InstalledVersions
array (
0 => '9999999-dev',
),
'reference' => '5ad152c70376002f043bb936d8ae5eed103fb993',
'reference' => '8ac5abdd497b37d2be4833bcf18d6819bba4d9c9',
),
'league/uri' =>
array (
@ -74,10 +74,11 @@ class InstalledVersions
),
'masterminds/html5' =>
array (
'pretty_version' => '2.7.5',
'version' => '2.7.5.0',
'pretty_version' => 'dev-master',
'version' => 'dev-master',
'aliases' =>
array (
0 => '2.7.x-dev',
),
'reference' => 'f640ac1bdddff06ea333a920c95bbad8872429ab',
),

2
plugins/af_readability/vendor/composer/autoload_psr4.php vendored
View File

@ -8,7 +8,7 @@ $baseDir = dirname($vendorDir);
return array(
'fivefilters\\Readability\\' => array($vendorDir . '/fivefilters/readability.php/src'),
'Psr\\Log\\' => array($vendorDir . '/psr/log/Psr/Log'),
'Psr\\Http\\Message\\' => array($vendorDir . '/psr/http-message/src', $vendorDir . '/psr/http-factory/src'),
'Psr\\Http\\Message\\' => array($vendorDir . '/psr/http-factory/src', $vendorDir . '/psr/http-message/src'),
'Masterminds\\' => array($vendorDir . '/masterminds/html5/src'),
'League\\Uri\\' => array($vendorDir . '/league/uri/src', $vendorDir . '/league/uri-interfaces/src'),
);

4
plugins/af_readability/vendor/composer/autoload_static.php vendored
View File

@ -37,8 +37,8 @@ class ComposerStaticInitb44cc79a0eaef9cd9c2f2ac697cbe9c0
),
'Psr\\Http\\Message\\' =>
array (
0 => __DIR__ . '/..' . '/psr/http-message/src',
1 => __DIR__ . '/..' . '/psr/http-factory/src',
0 => __DIR__ . '/..' . '/psr/http-factory/src',
1 => __DIR__ . '/..' . '/psr/http-message/src',
),
'Masterminds\\' =>
array (

33
plugins/af_readability/vendor/composer/installed.json vendored
View File

@ -7,14 +7,14 @@
"source": {
"type": "git",
"url": "https://dev.tt-rss.org/fox/readability-php.git",
"reference": "5ad152c70376002f043bb936d8ae5eed103fb993"
"reference": "8ac5abdd497b37d2be4833bcf18d6819bba4d9c9"
},
"require": {
"ext-dom": "*",
"ext-mbstring": "*",
"ext-xml": "*",
"league/uri": "^6.4",
"masterminds/html5": "^2.0",
"masterminds/html5": "2.7.x-dev@dev",
"php": ">=7.3.0",
"psr/log": "^1.0"
},
@ -25,7 +25,7 @@
"suggest": {
"monolog/monolog": "Allow logging debug information"
},
"time": "2022-07-16T13:23:08+00:00",
"time": "2022-07-31T06:02:47+00:00",
"default-branch": true,
"type": "library",
"installation-source": "source",
@ -241,19 +241,13 @@
},
{
"name": "masterminds/html5",
"version": "2.7.5",
"version_normalized": "2.7.5.0",
"version": "dev-master",
"version_normalized": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/Masterminds/html5-php.git",
"url": "https://dev.tt-rss.org/fox/html5-php.git",
"reference": "f640ac1bdddff06ea333a920c95bbad8872429ab"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Masterminds/html5-php/zipball/f640ac1bdddff06ea333a920c95bbad8872429ab",
"reference": "f640ac1bdddff06ea333a920c95bbad8872429ab",
"shasum": ""
},
"require": {
"ext-ctype": "*",
"ext-dom": "*",
@ -264,19 +258,24 @@
"phpunit/phpunit": "^4.8.35 || ^5.7.21 || ^6 || ^7"
},
"time": "2021-07-01T14:25:37+00:00",
"default-branch": true,
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "2.7-dev"
}
},
"installation-source": "dist",
"installation-source": "source",
"autoload": {
"psr-4": {
"Masterminds\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"autoload-dev": {
"psr-4": {
"Masterminds\\HTML5\\Tests\\": "test/HTML5"
}
},
"license": [
"MIT"
],
@ -297,18 +296,14 @@
"description": "An HTML5 parser and serializer.",
"homepage": "http://masterminds.github.io/html5-php",
"keywords": [
"HTML5",
"dom",
"html",
"html5",
"parser",
"querypath",
"serializer",
"xml"
],
"support": {
"issues": "https://github.com/Masterminds/html5-php/issues",
"source": "https://github.com/Masterminds/html5-php/tree/2.7.5"
},
"install-path": "../masterminds/html5"
},
{

11
plugins/af_readability/vendor/composer/installed.php vendored
View File

@ -6,7 +6,7 @@
'aliases' =>
array (
),
'reference' => '5006c754c42a09f5b88b258c2da4b9eff7508357',
'reference' => '4aefbd628e9a0e1eac58523904ad887b0635cda3',
'name' => '__root__',
),
'versions' =>
@ -18,7 +18,7 @@
'aliases' =>
array (
),
'reference' => '5006c754c42a09f5b88b258c2da4b9eff7508357',
'reference' => '4aefbd628e9a0e1eac58523904ad887b0635cda3',
),
'fivefilters/readability.php' =>
array (
@ -28,7 +28,7 @@
array (
0 => '9999999-dev',
),
'reference' => '5ad152c70376002f043bb936d8ae5eed103fb993',
'reference' => '8ac5abdd497b37d2be4833bcf18d6819bba4d9c9',
),
'league/uri' =>
array (
@ -50,10 +50,11 @@
),
'masterminds/html5' =>
array (
'pretty_version' => '2.7.5',
'version' => '2.7.5.0',
'pretty_version' => 'dev-master',
'version' => 'dev-master',
'aliases' =>
array (
0 => '2.7.x-dev',
),
'reference' => 'f640ac1bdddff06ea333a920c95bbad8872429ab',
),

42
plugins/af_readability/vendor/fivefilters/readability.php/.github/workflows/main.yml vendored Normal file
View File

@ -0,0 +1,42 @@
# This is a basic workflow to help you get started with Actions
name: CI
# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the master branch
push:
branches: [ master ]
pull_request:
branches: [ master ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# This workflow contains a single job called "build"
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest
strategy:
matrix:
php: ['7.3', '7.4', '8']
libxml: ['2.9.4', '2.9.5', '2.9.10', '2.9.12']
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
# Runs a single command using the runners shell
#- name: Run a one-line script
# run: echo Hello, world!
# Runs a set of commands using the runners shell
- name: Run a multi-line script
run: |
composer install
docker build --build-arg PHP_VERSION=${{matrix.php}} --build-arg LIBXML_VERSION=${{matrix.libxml}} -t gh-action - < ./docker/php/Dockerfile
docker run --volume $PWD:/app --workdir="/app" --env XDEBUG_MODE=coverage gh-action php ./vendor/bin/phpunit --coverage-clover /app/test/clover.xml

9
plugins/af_readability/vendor/fivefilters/readability.php/composer.json vendored
View File

@ -5,6 +5,13 @@
"keywords": ["readability", "html"],
"homepage": "https://github.com/fivefilters/readability.php",
"license": "Apache-2.0",
"repositories": [
{
"name": "masterminds/html5",
"type": "vcs",
"url": "https://dev.tt-rss.org/fox/html5-php.git"
}
],
"authors": [
{
"name": "Andres Rey",
@ -32,7 +39,7 @@
"ext-xml": "*",
"ext-mbstring": "*",
"psr/log": "^1.0",
"masterminds/html5": "^2.0",
"masterminds/html5": "2.7.x-dev@dev",
"league/uri": "^6.4"
},
"require-dev": {

56
plugins/af_readability/vendor/fivefilters/readability.php/docker/php/build.Dockerfile vendored Normal file
View File

@ -0,0 +1,56 @@
# Use this file to build a Docker image using the versions of PHP and Libxml specified.
# We have pre-built images at https://hub.docker.com/r/fivefilters/php-libxml which are faster to load than building from this file.
# To build using this file, use the following command from the root project folder (replace version of PHP/Libxml with the ones you want to use):
# docker build --build-arg PHP_VERSION=7.4 --build-arg LIBXML_VERSION=2.9.12 -t php-libxml -f ./docker/php/Dockerfile .
# To upload the image to Docker Hub, the tag (-t) value should be something like org/repo:tag, e.g. for us, fivefilters/php-libxml:php-8-libxml-2.9.12
# The tag can be applied afterwards too, e.g. docker tag php-libxml org/repo:tag
ARG PHP_VERSION=8
FROM php:${PHP_VERSION}-cli
# Install sqlite and libonig-dev (required for building PHP 7.4)
RUN apt-get update && apt-get install -y libsqlite3-dev libonig-dev
# Install libsodium (package doesn't work for some reason)
RUN curl https://download.libsodium.org/libsodium/releases/LATEST.tar.gz -o /tmp/libsodium.tar.gz && \
cd /tmp && \
tar -xzf libsodium.tar.gz && \
cd libsodium-stable/ && \
./configure && \
make && make check && \
make install
# Install custom version of libxml2
RUN apt-get install -y automake libtool unzip libssl-dev
# Remove current version
RUN apt-get remove -y libxml2
# Download new version, configure and compile
ARG LIBXML_VERSION=2.9.12
RUN curl https://gitlab.gnome.org/GNOME/libxml2/-/archive/v$LIBXML_VERSION/libxml2-v$LIBXML_VERSION.zip -o /tmp/libxml.zip && \
cd /tmp && \
unzip libxml.zip && \
cd libxml2-v$LIBXML_VERSION && \
./autogen.sh --libdir=/usr/lib/x86_64-linux-gnu && \
make && \
make install
# Recompile PHP with the new libxml2 library
RUN docker-php-source extract && \
cd /usr/src/php && \
./configure \
--with-libxml \
--enable-mbstring \
--with-openssl \
--with-config-file-path=/usr/local/etc/php \
--with-config-file-scan-dir=/usr/local/etc/php/conf.d && \
make && make install && \
docker-php-source delete
RUN apt-get update
#RUN pecl install libsodium
# Check if there's a pinned version of Xdebug for compatibility reasons
ARG XDEBUG_VERSION
RUN pecl install xdebug$(if [ ! ${XDEBUG_VERSION} = '' ]; then echo -${XDEBUG_VERSION} ; fi) && docker-php-ext-enable xdebug
# Required by coveralls
RUN apt-get install git -y

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/archive-of-our-own/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/archive-of-our-own/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Organization for Transformative Works",
"Direction": null,
"Excerpt": "An Archive of Our Own, a project of the Organization for Transformative Works",
"Image": null,
"Title": "Conversations with a Cryptid - Chapter 1 - AMournfulHowlInTheNight - \u50d5\u306e\u30d2\u30fc\u30ed\u30fc\u30a2\u30ab\u30c7\u30df\u30a2 | Boku no Hero Academia",
"SiteName": null
}

317
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/archive-of-our-own/expected.html vendored Normal file
View File

@ -0,0 +1,317 @@
<div role="article" id="chapters">
<h3 id="work">
Chapter Text
</h3>
<p>
Izuku was struggling to understand how he had even managed to get here, seated before the archvillain of Japan with only a sense of dread to keep him company. All Might sat concealed in an observation room, of the firm opinion that he could only aggravate the prisoner and he sent Izuku off with a strained smile. A vague haze hovered over Izuku’s memory. It started with a simple conversation gone astray on a long drive home.
</p>
<p>
“So, who is All For One? Do we know anything about him beyond what you told me before? He’s been imprisoned for months now.” Izuku remembered asking All Might from the backseat of the car as Detective Tsukauchi leisurely drove along a sprawling highway.
</p>
<p>
Playing on the car radio was an aftermath report of a villain attack in downtown Tokyo. Izuku caught the phrase “liquid body” from the female reporter before Detective Tsukauchi changed the channel.
</p>
<p>
“Nope. Still nothing. No one really wants to speak to him,” All Might had replied brightly. “He gives off polite airs, but he’s a piece of work.” All Might’s mostly obstructed shoulders in the front seat shrugged. “Not much you can do with someone like him. Everything that comes out is a threat or taunt.” All Might carefully waved his hand in a circular motion towards the side of his head.
</p>
<p>
“No one’s even made it through a full interview with him, from what I’ve heard,” Detective Tsukauchi added from behind the wheel. “He plays mind games with them. The prison also has a “no recent events” policy on any discussions with him as well. Just in case he ends up with ideas or has some means of communicating. Given that people only want to ask him about current events, it doesn’t leave much to talk about.”
</p>
<p>
“Wait, they still don’t know what Quirks he has?” Izuku asked exasperatedly. “They can’t if there’s still an information block on visits.”
</p>
<p>
“Nope. We have no idea what he can do. They can run DNA tests, but it’s not like anyone apart from him even knows how his Quirk works. They could get matches with any number of people, but if they’re not in a database then we can’t cross-reference them anyway. Even if they run an analysis, the data doesn’t mean anything without the ability to interpret it,” All Might gestured with a skeletal finger. “It’s a waste of time after the initial tests were conducted. They weren’t game to MRI him either, given he’s definitely got a Quirk that creates metal components.”
</p>
<p>
“No one’s bothered to ask him anything about… anything?” Izuku asked, dumbfounded. “He must be around two-hundred years old and people can’t think of a single non-current affairs thing to ask him?”
</p>
<p>
In some ways it was unfathomable that they’d let a potential resource go to waste. On the other hand, said potential resource had blown up a city, murdered numerous people and terrorised Japan for over a century. At the very least.
</p>
<p>
“Well, I tried to ask him about Shigaraki, but he didn’t say much of anything really. Some garbage about you being too dependent on me and him letting Shigaraki run wild and how he just wanted to be the ultimate evil,” All Might shrugged again. “He spends too much time talking about nothing.”
</p>
<p>
Izuku shifted his head onto his arm. “But, that’s not really nothing, is it?”
</p>
<p>
“What do you mean?” Izuku had the feeling that All Might would have been looking at him with the <i>you’re about to do something stupid aren’t you</i> expression that was thankfully becoming less common.
</p>
<p>
“Well, he clearly doesn’t know anything about us, All Might, if he thinks that you’re just going to let go of me after not even two years of being taught. Maybe Shigaraki was dependent on adult figures, but I don’t even remember my dad and mum’s been busy working and keeping the house together. I’ve never had a lot of adult supervision before,” Izuku laughed nervously. “I had to find ways to keep myself entertained. If anything, I’m on the disobedient side of the scale.” All Might outright giggled.
</p>
<p>
“I’ll say, especially after what happened with Overhaul. I’m surprised your mother let you leave the dorms again after that.”
</p>
<p>
“I’m surprised she didn’t withdraw and ground me until I was thirty.”
</p>
<p>
“Oh? That strict?” Tsukauchi asked.
</p>
<p>
“She has her moments,” Izuku smiled fondly. “Do you think she’d agree to me asking the archvillain of Japan about his Quirk?” Izuku asked, only partially joking. There was an itch at the back of his head, a feeling of something missing that poked and prodded at his senses.
</p>
<p>
All Might coughed and sprayed the dash with a fine red mist. “Absolutely not! I forbid it!”
</p>
<p>
“That’s exactly why I’m asking her and not you,” Izuku grinned from the backseat.
</p>
<p>
“He’s evil!”
</p>
<p>
“He’s ancient. You honestly don’t wonder about the sort of things someone with that life experience and Quirk would have run across to end up the way he did?”
</p>
<p>
“Nope, he made it perfectly clear that he always wanted to be the supreme evil,” All Might snipped through folded arms.
</p>
<p>
“Yeah, and I’ll just take his word for that, won’t I?” Izuku grinned. “If he does nothing but lie, then that’s probably one too, but there’s a grain of truth in there somewhere.”
</p>
<p>
“What would you even do? Harass him into telling you his life story?” All Might sighed.
</p>
<p>
“Not when I can kill him with kindness. Who knows, it might even be poisonous for him.”
</p>
<p>
“You’re explaining this to your mother. Teacher or not, I’m not being on the receiving end of this one.”
</p>
<p>
Izuku blinked for a moment. “You’ll let me?”
</p>
<p>
“I’m not entirely for it, but any prospective information on what influenced Shigaraki can only be a good thing. If anything goes south we can pull you out pretty easily. Just be aware of who and what you’re dealing with.” Struggling, All Might turned a serious look to Izuku around the side of the seat. “<i>Only</i> if your mother gives the okay.”
</p>
<p>
The conversation turned to school for the rest of the way.
</p>
<p>
It might have been curiosity or it might have been the nagging sensation that chewed at his brain for the three weeks that he researched the subject of the conversation. All For One was a cryptid. Mystical in more ways than one, he was only a rumour on a network that was two-hundred years old. There were whispers of a shadowy figure who once ruled Japan, intermingled with a string of conspiracies and fragmented events.
</p>
<p>
Izuku had even braved the dark web, poking and prodding at some of the seedier elements of the world wide web. The internet had rumours, but the dark web had stories.<br>
</p>
<p>
An implied yakuza wrote about his grandfather who lost a fire manipulation Quirk and his sanity without any reason. His grandfather had been institutionalised, crying and repeating “he took it, he took it” until his dying days. No one could console him.
</p>
<p>
Another user spoke of a nursing home where a room full of dementia residents inexplicably became docile and no longer used their Quirks on the increasingly disturbed staff. The nursing home erupted into flames just before a court case against them commenced.
</p>
<p>
A user with neon pink text spoke of how their great-great-great-great grandmother with a longevity Quirk had simply aged rapidly one day and passed away in her sleep, her face a mask of terror. No cause had ever been found.
</p>
<p>
A hacker provided a grainy CCTV recording of a heist and a scanned collection of documents from over a century ago, where there was a flash of light and entire bank vault had been emptied. What separated it from the usual robbery was that it contained a list containing confidential information on the Quirks of the First Generation. Izuku had greedily snavelled up and saved the video and documents to an external hard drive.
</p>
<p>
Paging through, Izuku saw someone recount how their Quirkless uncle had developed a warp Quirk and gone from rags to riches under a mysterious benefactor. A decade ago, the uncle had simply disappeared.
</p>
<p>
Numerous and terrifying, the stories were scattered nuggets of gold hidden across the web. They’d never last long, vanishing within hours of posting. Izuku bounced from proxy to proxy, fleeing from a series of deletions that seemed to follow Izuku’s aliased postings across snitch.ru, rabbit.az, aconspiracy.xfiles and their compatriots.
</p>
<p>
After thirty-two identity changes (all carefully logged in a separate notebook), a large amount of feigning communal interest in a lucky tabloid article on All For One which had been released at the start of the first of the three weeks, Izuku hung up his tinfoil hat and called it a month. He haphazardly tossed a bulging notebook into his bookshelf and lodged his hard drive in a gap containing seven others and went to dinner.
</p>
<p>
It took another week to present his research to All Might and Tsukauchi, whose jaws reached the proverbial floor.
</p>
<p>
“We never found any of this,” the Detective Tsukauchi exclaimed. “How did you find all of it?”
</p>
<p>
“I asked the right people. Turns out criminals have very long and very unforgiving memories,” Izuku explained through sunken eyes. “There’s more than this that could be linked to him, but these ones seem to be the most obvious.”
</p>
<p>
“They would do, you can’t be head of the underworld without making an army of enemies,” All Might agreed. “You know, if you can get any more information about these events, I think you’ll give people a lot of peace of mind.”
</p>
<p>
“Provided mum agrees to it.”
</p>
<p>
“Only if she agrees to it.”
</p>
<p>
It took another month to convince his mother, who eventually gave in once All Might provided an extremely comprehensive schedule of how the visitations and any resulting research would be carefully balanced against Izuku’s schoolwork and internship.
</p>
<p>
The day of the visit finally arrived, four months after the initial conversation, much to Izuku’s dismay.
</p>
<p>
Izuku remembered how he had arrived, with the Detective and All Might escorting him through its sterile, white innards. A list of rules rattled off at the gate, “no current affairs” was chief among them and an assertion that he’d be dragged from the room if need be if Izuku was to breach any of them. No smuggling of communication devices, no weapons, no Quirks, nothing that could compromise the prisoner’s secure status.
</p>
<p>
Heavily armoured and drilled guards leading him underground into the deepest bowels of the Tartarus complex.
</p>
<p>
Izuku understood the rules, dressed casually in a cotton t-shirt with “Shirt” printed across it in haphazard English and clutching at a carefully screened and utterly blank notebook.
</p>
<p>
Across from him, behind reinforced glass, the archvillain of Japan was bound and unmoving.
</p>
<p>
“Hello,” Izuku initiated uncertainly. His skin had been crawling the moment he crossed the threshold, a memory of the encounter and escape at the Kamino Ward months ago.
</p>
<p>
“Ah, All Might’s disciple,” drawled All For One, “is he too cowardly to come himself? Yet I don’t hear the garments of a hero.” With hardly a word out, All For One had already lunged for the figurative jugular.
</p>
<p>
A stray thought of <i>how does he know who I am if he’s blind and isn’t familiar with me?</i> whispered its way through Izuku’s head.
</p>
<p>
“Oh, no,” Izuku corrected hastily, almost relieved at the lack of any pretence, “I asked if I could talk to you. This isn’t exactly hero related.”
</p>
<p>
“I’m surprised he said yes.” While there was little by way of expression, Izuku could just about sense the contempt dripping from the prisoner’s tone. It wasn’t anything he wasn’t expecting. Kacchan had already said worse to him in earlier years. Water off a duck’s back.
</p>
<p>
“Well, he’s not my legal guardian, so I think you should be more surprised that mum said yes. She’s stricter with these things than All Might,” Izuku corrected again. “Mum gave the okay, but that was a stressful discussion.” And there it was, a miniscule twitch from the man opposite. A spasm more than anything else. <i>Interesting.</i> Pinned down as he was, the prisoner oozed irritation.
</p>
<p>
“At least your mother is a wise person. I wonder why the student doesn’t heed all of the advice of the teacher.” All For One’s tone didn’t indicate a question, so much as an implicit statement that All Might wasn’t worth listening to in any capacity. Kacchan would have hated the comparison, but the hostility had an almost comfortable familiarity. “He no doubt warned you off speaking to me, overprotective as he is, but here you are.”
</p>
<p>
Izuku found himself smiling at the thought of Kacchan’s outrage if he ever found out about the mental comparison as he replied. “I don’t think it’s normal for anyone my age to listen completely to their teachers. We pick and choose and run with what works best for us. He warned me, but I’m still here. Mum warned me as well, but I think she cared more about the time management aspect of it."
</p>
<p>
“Is that a recent development?” All For One probed.
</p>
<p>
“Not really. My old homeroom teacher told me not to bother applying to U.A.” His mother’s beaming face had carried Izuku through the cheerful and resolute signing of that application form.
</p>
<p>
“I see you followed their advice to the letter,” came the snide, dismissive reply.
</p>
<p>
Izuku hoisted up his legs and sat cross-legged in his seat. Leaning slightly forward as he did so as to better prop up his notebook.
</p>
<p>
“You’re a walking contrarian, aren’t you? All Might told me about his run ins with you. What someone does or doesn’t do really doesn’t matter to you, you’ll just find a way to rationalise it as a negative and go on the attack anyway. What you’re currently doing is drawing attention away from yourself and focusing it on me so you can withhold information.” Izuku flipped open his notebook and put pen to paper. “You’ve got something fairly big to hide and you diverting attention exposes that motivation as existing anyway. The only real questions here are what and why?” Izuku paused in mortification as the man opposite’s lips parted. “I just said that aloud, didn’t I?”
</p>
<p>
Of the responses Izuku had expected, it wasn’t laughter. Unrestrained, Izuku would have expected a violent outburst. In this situation, he would have expected another scathing comment. Instead, All For One laughed breathily, leaning into his bonds. Wheezingly he spoke, “I’ll have to change tactics, if that one’s too transparent for you. How refreshing.”
</p>
<p>
Doing his best not to glow a blinding red and simultaneously pale at the interest, Izuku carried on. “I add it to the list when you do. I’m not emotionally involved enough to really be impacted by what you’re saying. I know about you in theory, but that’s it. Maybe All Might has a history with you, but I don’t really know enough about you personally to…”
</p>
<p>
“Care,” All For One supplied, somewhat subdued as he struggled to breathe. “You’re only here to satisfy your curiosity as to whether or not the stories were true.”
</p>
<p>
Izuku nodded, scratching at his notebook with his left hand. “Yes and no, I’m actually here to ask you about how your Quirk works.” <i>For now.</i>
</p>
<p>
Another chortle, more restrained that the last.
</p>
<p>
"What makes you think others haven’t already asked?” Had All For One been unrestrained, Izuku could imagine the stereotypical scene of the villain confidently leaning back in some overblown chair in a secret lair, drink of choice in hand, if the tone of voice was any indication. Deflections aside, the man easily rose to each comment.
</p>
<p>
“Whether or not they asked it’s irrelevant if they can’t read the answers.” Answers didn’t matter if the people involved were too attached to read into the answers. If none of the interviewers had managed a full interview, then it seemed unlikely that any sort of effort was put into understanding the villain.
</p>
<p>
“And you think you can? What expertise do you hold above theirs?” Doubt and reprimand weighted the words. Oddly enough, had Izuku been any younger he could have mistaken the man for a disapproving parent rebuking an overly ambitious child. Albeit an extremely evil one.
</p>
<p>
Izuku inhaled shortly and went for it. “If there’s something I know, it’s Quirks and how they work. Maybe I don’t know you, but I don’t really need to. Quirks fall under broad categories of function. You can take and give, consent doesn’t seem to be a factor. You either can’t “see” certain types of Quirks or you need to have prior knowledge of it before you take it with what I know about your brother. Despite your <i>nom de guerre</i>, because we both know it’s not your real name, you have a history of giving multiple Quirks and causing brain damage to the receiver. You clearly aren’t impacted by those same restrictions, so it must either alter your brain mapping or adjust functions to allow for simultaneous use and storage. It also must isolate or categories the Quirks you stock, because from the few people who do remember you, you creating certain Quirks is always in the context of giving them to someone else meaning there’s probably an inherent immunity to stop it from tainting your own Quirk with a mutation,” Izuku mumbled, almost to himself. “The only thing really in question about your Quirk is the finer details and whether or not you need to maintain those features or if they’re inherent and your hard limit for holding Quirks.”
</p>
<p>
There was silence, for only a moment. “If only my hands were free, I would clap for such a thoughtful assessment. Clearly you’re not all brawn,” All For One positively purred. “Speculate away.” A wide and slightly unhinged smile was directed at Izuku.
</p>
<p>
It was all Izuku could do not to wince at the eagerness. An image of a nervous All Might, hidden in the observation room above with the grim-faced prison staff, came to mind.
</p>
<p>
“I note that you said thoughtful and not correct,” and Izuku breathed and unsteadily jotted it down in his notebook. “You don’t seem bothered by the guess.”
</p>
<p>
“Few people live long enough to question my Quirk, let alone have the talent to guess so thoughtfully at its functions. It seems we share a hobby.” There was something terribly keen in that voice that hadn’t been there before, twisting itself through the compliment.
</p>
<p>
“I suppose it helps that you’re playing along out of boredom,” Izuku verbally dodged, unease uncoiling itself from the back of his mind.
</p>
<p>
“I <i>was</i> playing along out of boredom,” All For One corrected smoothly. “Now, I’m curious. Admittedly, my prior assumptions of you weren’t generous, but I’ve been too hasty in my assessments before.”
</p>
<p>
“I’ll pack up and leave now if that’s the case,” Izuku replied with only half an ear on the conversation as the words on his page began to drastically expand to distract himself from the building anxiety.
</p>
<p>
“Sarcasm, so you do have characteristics of a normal teenager. Your willingness to maim yourself has often left me wondering…”
</p>
<p>
“You’re deflecting again,” Izuku observed. “I’m not sure if that’s a nervous habit for you or if you’re doing it because I’m close to being right about your Quirk. That being said, I don’t think you know what a normal teenager is if Shigaraki is any indication. He’s about seven years too late for his rebellious phase.”
</p>
<p>
“I’m hurt and offended,” came the amused reply.
</p>
<p>
“By how Shigaraki ended up or your parenting? You only have yourself to blame for both of them.”
</p>
<p>
“How harsh. Shigaraki is a product of society that birthed him. I can’t take credit for all of the hard work,” All For One laid out invitingly. Perhaps someone else would have risen to the bait, but Izuku was already packing his mental bags and heading for the door.
</p>
<p>
Clearly the prisoner’s anticipation had registered poorly with someone in the observation room, because a voice rang through the air. “Time’s up Midoriya-kun.”
</p>
<p>
“Okay!” Izuku called back and etched out his last thoughtful of words, untangled his legs and rose to his feet.
</p>
<p>
“What a shame, my visitations are always so short,” All For One spoke mournfully.
</p>
<p>
“Well, you did blow up half a city. They could have just let you suffocate instead. Same time next week, then?” Izuku offered brightly, notebook stuffed into a pocket and was followed out the door by wheezing laughter.
</p>
<p>
It was only after he had made it safely back to the communal room where All Might waited did he allow the spring to fade from his step and discard his nervous smile. Shuddering, he turned to All Might whose face was set in a grimace.
</p>
<p>
“I won’t say I told you so,” All Might offered, perched on the edge of his couch like a misshapen vulture.
</p>
<p>
“He’s… not really what I was expecting. I was expecting someone, more openly evil.” Izuku allowed himself to collapse into the leather of the seat. He shakily reached for the warm tea that had been clearly been prepared the moment Izuku left the cell. “I suppose he does it to lull people into a false sense of security. I didn’t understand how someone with only half a set of expressions could have “villain” written all over them until I met him.”
</p>
<p>
“He’s always been like that. He feigns concern and sympathy to lure in society’s outcasts. They’re easy targets,” All Might said through a mouthful of biscuit.
</p>
<p>
“Has he ever tried it on any of the One For All successors?”
</p>
<p>
“Not really, but you might have accidentally given him the incentive for it. He never had access to any of the One For All wielders while they were young.” All Might snorted, “not that it’ll make a difference with you”.
</p>
<p>
“I think he was trying to gauge me for a world view before the wardens ended it. I need more time to work out his response to the stuff on his Quirk.”
</p>
<p>
“He’s conversation starved since it’s solitary confinement. If what the people monitoring his brain activity said was true, you’re the most exciting thing to have happened to him in months. He replied after you left, said he was looking forward to it.”
</p>
<p>
“That’s pretty sad."
</p>
<p>
“It’s even sadder that we’re the only two members of the public who have had anything to do with him. Stain gets a pile of mail from his “fans”, but All For One has nothing,” All Might waved a tea spoon. “That’s what he gets.”
</p>
<p>
“Let’s get out of here and tell Detective Tsukauchi how it went.” Izuku gulped down his tea and headed for the exit, with him and All Might reaching it at roughly the same amount of time.
</p>
<p>
“At least your mum’s making katsudon for us tonight," was All Might's only optimistic comment.
</p>
<p>
Anxiety was still ebbing over Izuku after Tsukauchi had been debriefed in the car.
</p>
<p>
<i>“It seems we share a hobby.”</i> Haunted Izuku on the drive home. As if ripping someone’s Quirk from them and leaving them lying traumatised on the ground was just a fun pastime and not an act of grievous bodily harm.
</p>
<p>
And he’d be dealing with him again in another week.
</p>
</div>

1025
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/archive-of-our-own/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

6
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/citylab-1/expected-images.json vendored Normal file
View File

@ -0,0 +1,6 @@
[
"https:\/\/cdn.citylab.com\/media\/img\/citylab\/2019\/04\/mr1\/facebook.jpg?1556645448",
"https:\/\/cdn.citylab.com\/media\/img\/citylab\/2019\/04\/mr1\/300.jpg?mod=1556645448",
"https:\/\/cdn.theatlantic.com\/assets\/media\/img\/posts\/2019\/04\/AP_8912060228\/cbd32b0e1.jpg",
"https:\/\/cdn.theatlantic.com\/assets\/media\/img\/posts\/2019\/04\/AP_945361213236\/888fdd750.jpg"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/citylab-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Sarah Archer",
"Direction": null,
"Excerpt": "The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline.",
"Image": "https:\/\/cdn.citylab.com\/media\/img\/citylab\/2019\/04\/mr1\/facebook.jpg?1556645448",
"Title": "The Modern Ambitions Behind Neon",
"SiteName": "CityLab"
}

108
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/citylab-1/expected.html vendored Normal file
View File

@ -0,0 +1,108 @@
<article itemscope="itemscope" itemtype="https://schema.org/NewsArticle" xmlns:xlink="http://www.w3.org/1999/xlink">
<meta itemprop="datePublished" content="2019-04-30T13:39:00-04:00">
<meta itemprop="dateModified" content="2019-04-30T13:40:00-04:00">
<meta itemprop="mainEntityOfPage" content="https://www.citylab.com/design/2019/04/neon-signage-20th-century-history/588400/">
<figure itemprop="image" itemscope="itemscope" itemtype="http://schema.org/ImageObject">
<picture><source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/940.jpg?mod=1556645448" media="(min-width: 1024px)"> <source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/lead_large.jpg?mod=1556645448" media="(min-width: 576px)"></picture>
<meta itemprop="height" content="128">
<meta itemprop="width" content="300">
<meta itemprop="url" content="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"><picture><source srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" media="(max-width: 575px)"><img src="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448" alt srcset="https://cdn.citylab.com/media/img/citylab/2019/04/mr1/300.jpg?mod=1556645448"></picture>
<figcaption>
<span itemprop="caption">The Moulin Rouge cabaret in Paris</span> <span itemprop="creator">Benoit Tessier/Reuters</span>
</figcaption>
</figure>
<div>
<h2 itemprop="headline">
Why Neon Is the Ultimate Symbol of the 20th Century
</h2>
</div>
<h2 itemprop="description">
The once-ubiquitous form of lighting was novel when it first emerged in the early 1900s, though it has since come to represent decline.
</h2>
<section id="article-section-1">
<p>
In the summer of 1898, the Scottish chemist Sir William Ramsay made a discovery that would eventually give the Moulin Rouge in Paris, the Las Vegas Strip, and New York’s Times Square their perpetual nighttime glow. Using the boiling point of argon as a reference point, Ramsay and his colleague Morris W. Travers isolated three more noble gases and gave them evocative Greek names: neon, krypton, and xenon. In so doing, the scientists bestowed a label of permanent novelty on the most famous of the trio—neon, which translates as “new.” This discovery was the foundation on which the French engineer Georges Claude crafted a new form of illumination over the next decade. He designed glass tubes in which neon gas could be trapped, then electrified, to create a light that glowed reliably for more than 1,000 hours.
</p>
<p>
In the 2012 book <em>L’être et le Néon</em>, <a href="https://mitpress.mit.edu/books/being-and-neonness-translation-and-content-revised-augmented-and-updated-edition-luis-de-miranda">which has been newly translated into English by Michael Wells</a>, the philosopher Luis de Miranda weaves a history of neon lighting as both artifact and metaphor. <em>Being and Neonness</em>, as the book is called in its English edition, isn’t a typical material history. There are no photographs. Even de Miranda’s own example of a neon deli sign spotted in Paris is re-created typographically, with text in all caps and dashes forming the border of the sign, as one might attempt on Twitter. Fans of Miami Beach’s restored Art Deco hotels and California’s bowling alleys might be disappointed by the lack of glossy historical images. Nonetheless, de Miranda makes a convincing case for neon as a symbol of the grand modern ambitions of the 20th century.
</p>
<p>
De Miranda beautifully evokes the notion of neon lighting as an icon of the 1900s in his introduction: “When we hear the word <em>neon</em>, an image pops into our heads: a combination of light, colors, symbols, and glass. This image is itself a mood. It carries an atmosphere. It speaks … of the essence of cities, of the poetry of nights, of the 20th century.” When neon lights debuted in Europe, they seemed dazzlingly futuristic. But their husky physicality started becoming obsolete by the 1960s, thanks in part to the widespread use of plastic for fluorescent signs. Neon signs exist today, though they’ve been eclipsed by newer technologies such as digital billboards, and they remain charmingly analog: Signs must be made by hand because there’s no cost-effective way to mass-produce them.
</p>
<p>
In the 1910s, neon started being used for cosmopolitan flash in Paris at precisely the time and place where the first great modernist works were being created. De Miranda’s recounting of the ingenuity emerging from the French capital a century ago is thrilling to contemplate: the cubist art of Pablo Picasso, the radically deconstructed fashions of Coco Chanel, the stream-of-consciousness poetry of Gertrude Stein, and the genre-defying music of Claude Debussy—all of which heralded a new age of culture for Europe and for the world.
</p>
</section>
<section id="article-section-2">
<p>
Amid this artistic groundswell, Georges Claude premiered his neon lights at the <a href="https://www.mondial-paris.com/en/visiteur/auto">Paris Motor Show</a> in December 1910, captivating visitors with 40-foot-tall tubes affixed to the building’s exterior. The lights shone orange-red because neon, by itself, produces that color. <em>Neon lighting</em> is a catchall term that describes the technology of glass tubing that contains gas or chemicals that glow when electrified. For example, neon fabricators use carbon dioxide to make white, and mercury to make blue. Claude acknowledged at the time that neon didn’t produce the ideal color for a standard light bulb and insisted that it posed no commercial threat to incandescent bulbs.
</p>
<p>
Of course, the very quality that made neon fixtures a poor choice for interior lighting made them perfect for signs, de Miranda notes. The first of the neon signs was switched on in 1912, advertising a barbershop on Paris’s Boulevard Montmartre, and eventually they were adopted by cinemas and nightclubs. While Claude had a monopoly on neon lighting throughout the 1920s, the leaking of trade secrets and the expiration of a series of patents broke his hold on the rapidly expanding technology.
</p>
</section>
<section id="article-section-3">
<p>
In the following decades, neon’s nonstop glow and vibrant colors turned ordinary buildings and surfaces into 24/7 billboards for businesses, large and small, that wanted to convey a sense of always being open. The first examples of neon in the United States debuted in Los Angeles, where the Packard Motor Car Company commissioned two large blue-and-orange <span>Packard</span> signs that literally stopped traffic because they distracted motorists. The lighting also featured heavily at the Chicago Century of Progress Exposition in 1933 and at the 1939 World’s Fair in New York. At the latter event, a massive neon sign reading <span>Futurama</span> lit the way to a General Motors exhibition that heralded “The World of Tomorrow.”
</p>
<figure>
<picture><img alt data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_8912060228/cbd32b0e1.jpg"></picture>
<figcaption>
Workers remove a hammer and sickle from a neon sign that reads “Glory to Communism,” visible on the roof of the Communist-run electricity-board headquarters in Czechoslovakia in 1989. (AP)
</figcaption>
</figure>
<p>
De Miranda points out that businesses weren’t alone in embracing neon’s ability to spread messages effectively. By the middle of the century, the lighting was being adopted for more political purposes. “In the 1960s, the Soviets deployed a vast ‘neonization’ of the Eastern bloc capitals to emulate capitalist metropolises,” de Miranda writes. “Because consumer shops were rare in the Polish capital [of Warsaw], they did not hesitate to illuminate the façades of public buildings.” In other words, as opposed to the sole use of the more obvious forms of propaganda via posters or slogans, the mass introduction of neon lighting was a way of getting citizens of Communist cities to see their surroundings with the pizzazz and nighttime glamour of major Western capitals.
</p>
</section>
<section id="article-section-4">
<p>
Neon, around this time, began to be phased out, thanks to cheaper and less labor-intensive alternatives. In addition, the global economic downturn of the 1970s yielded a landscape in which older, flickering neon signs, which perhaps their owners couldn’t afford to fix or replace, came to look like symbols of decline. Where such signs were once sophisticated and novel, they now seemed dated and even seedy.
</p>
<section>
<h2>
Cities are changing fast. Keep up with the <b>CityLab Daily</b> newsletter.
</h2><label for="promo-email-input-email">The best way to follow issues you care about.</label>
</section>
<p>
De Miranda understands this evolution by zooming out and looking at the 1900s as the “neon century.” The author draws a parallel between the physical form of neon lights, which again are essentially containers for electrified gases, and that of a glass capsule—suggesting they are a kind of message in a bottle from a time before the First World War. “Since then, [neon lights] have witnessed all the transformations that have created the world we live in,” de Miranda writes. “Today, they sometimes seem to maintain a hybrid status, somewhere between junkyards and museums, not unlike European capitals themselves.”
</p>
<figure>
<picture><img alt data-srcset="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg" src="https://cdn.theatlantic.com/assets/media/img/posts/2019/04/AP_945361213236/888fdd750.jpg"></picture>
<figcaption>
Martin Wartman, a student at Northern Kentucky University, works on a neon sign at the Neonworks of Cincinnati workshop connected to the American Sign Museum, in 2016. (John Minchillo / AP)
</figcaption>
</figure>
<p>
Another mark of neon’s hybridity: Its obsolescence started just as some contemporary artists began using the lights in their sculptures. Bruce Nauman’s 1968 work <em><a href="https://www.stedelijk.nl/en/collection/1097-bruce-nauman-my-name-as-though-it-were-written-on-the-surface-of-the-moon">My Name as Though It Were Written on the Surface of the Moon</a></em> poked fun at the space race—another symbol of 20th-century technological innovation whose moment has passed. The piece uses blue “neon” letters (mercury, actually) to spell out the name “bruce” in lowercase cursive, with each character repeated several times as if to convey a person speaking slowly in outer space. The British artist Tracey Emin has made <a href="https://www.artsy.net/collection/tracey-emin-neon-sculptures-and-prints">sculptures</a> that resemble neon Valentine’s Day candies: They read as garish and sentimental confections with pink, heart-shaped frames that surround blue text fragments. Drawing on the nostalgia-inducing quality of neon, the sculptures’ messages are redolent of old-fashioned movie dialogue, with titles such as “You Loved Me Like a Distant Star” and “The Kiss Was Beautiful.”
</p>
<p>
Seeing neon lighting tamed in the context of a gallery display fits comfortably with de Miranda’s notion that neon technology is like a time capsule from another age. In museums, works of neon art and design coexist with objects that were ahead of their own time in years past—a poignant fate for a technology that made its name advertising “The World of Tomorrow.” Yet today neon is also experiencing a kind of craft revival. The fact that it can’t be mass-produced has made its fabrication something akin to a cherished artisanal technique. Bars and restaurants hire firms such as Let There Be Neon in Manhattan, or <a href="https://www.instagram.com/theneonqueen/">the L.A.-based master neon artist Lisa Schulte</a>, to create custom signs and works of art. Neon’s story even continues to glow from inside museums such as California’s <a href="https://www.neonmona.org/">Museum of Neon Art</a> and the Neon Museum in Las Vegas. If it can still be a vital medium for artists and designers working today, “neonness” need not only be trapped in the past. It might also capture the mysterious glow of the near future—just as it did a century ago.
</p>
<p>
<em>This article originally appeared on <a href="https://www.theatlantic.com/entertainment/archive/2019/04/being-and-neonness-neon-lights-symbol-20th-century/588184/">The Atlantic</a>.</em>
</p>
</section>
<section data-include="css:https://cdn.citylab.com/static/a/frontend/dist/citylab/css/components/author-article.cf4e8e0b143f.css">
<h4>
About the Author
</h4>
<div itemprop="author" itemscope="itemscope" itemtype="http://schema.org/Person">
<h5 itemprop="name">
<a href="https://www.citylab.com/authors/sarah-archer/">Sarah Archer</a>
</h5>
<p itemprop="description">
<a href="https://www.citylab.com/authors/sarah-archer/" data-omni-click="inherit">Sarah Archer</a> is the author of <em>The Midcentury Kitchen</em>.
</p>
</div>
</section>
</article>

683
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/citylab-1/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

6
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/data-url-image/expected-images.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/data-url-image/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Lorem ipsum dolor sit amet consectetur adipisicing elit. Natus eaque totam provident obcaecati nisi praesentium iusto velit fuga debitis quidem ut repellat corrupti, eligendi inventore quibusdam perspiciatis delectus omnis pariatur excepturi quasi fugit? A adipisci natus nostrum, qui aperiam, at culpa corrupti autem enim earum vitae. Nostrum et officiis facere ex recusandae tenetur, delectus odit provident soluta id perferendis ducimus quibusdam corporis rerum voluptatem architecto sequi beatae quod mollitia voluptatibus earum tempora inventore ut. Deserunt reprehenderit recusandae nostrum, eaque fuga cum, repellat, perspiciatis ducimus in non consequatur ratione. Sint rerum necessitatibus deleniti odio earum voluptatum eos modi ab dolor minus.",
"Image": null,
"Title": "Document",
"SiteName": null
}

11
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/data-url-image/expected.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

19
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/data-url-image/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

5
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/dropbox-blog/expected-images.json vendored Normal file
View File

@ -0,0 +1,5 @@
[
"https:\/\/aem.dropbox.com\/cms\/content\/dam\/dropbox\/tech-blog\/en-us\/2020\/11\/atf\/diagrams\/Techblog-ATF-Social.png",
"http:\/\/fakehost\/cms\/content\/dam\/dropbox\/tech-blog\/en-us\/2020\/11\/atf\/diagrams\/Techblog-ATF-720x844px-1.png",
"http:\/\/fakehost\/cms\/content\/dam\/dropbox\/tech-blog\/en-us\/2020\/11\/atf\/diagrams\/Techblog-ATF-720x225px-2.png"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/dropbox-blog/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Arun Sai Krishnan",
"Direction": null,
"Excerpt": "I joined Dropbox not long after graduating with a Master\u2019s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.",
"Image": "https:\/\/aem.dropbox.com\/cms\/content\/dam\/dropbox\/tech-blog\/en-us\/2020\/11\/atf\/diagrams\/Techblog-ATF-Social.png",
"Title": "How we designed Dropbox\u2019s ATF - an async task framework",
"SiteName": null
}

527
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/dropbox-blog/expected.html vendored Normal file
View File

@ -0,0 +1,527 @@
<div>
<div>
<p>
I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.
</p>
<p>
This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.
</p>
<p>
ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services.&nbsp;
</p>
<p>
As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.
</p>
<p>
Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.
</p>
</div>
<div>
<p id="introduction">
<h2>
Introduction
</h2>
</p>
</div>
<div>
<p>
Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.
</p>
<p>
Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling.&nbsp;
</p>
</div>
<div>
<p id="glossary">
<h2>
Glossary
</h2>
</p>
</div>
<div>
<p>
Some basic terms repeatedly used in this post, defined as used in the context of this discussion.
</p>
<p>
<b>Lambda:</b> A callback implementing business logic.
</p>
<p>
<span><b>Task:</b> Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.</span>
</p>
<p>
<span><b>Collection:</b> A labeled subset of tasks belonging to a lambda. If <span>send email</span> is implemented as a lambda, then <span>password reset email</span> and <span>marketing email</span> would be collections.</span>
</p>
<p>
<span><b>&nbsp;Priority:</b> Labels defining priority of execution of tasks within a lambda.&nbsp;</span>
</p>
</div>
<div>
<p id="features">
<h2>
Features
</h2>
</p>
</div>
<div>
<p>
<b>Task scheduling</b><br>
Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.
</p>
<p>
<b>Priority based execution</b><br>
Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.
</p>
<p>
<b>Task gating</b><br>
ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.
</p>
<p>
<b>Track task status</b><br>
Clients can query the status of a scheduled task.
</p>
</div>
<div>
<p id="system-guarantees">
<h2>
System guarantees
</h2>
</p>
</div>
<div>
<p>
<b>At-least once task execution<br></b> The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.
</p>
<p>
<b>No concurrent task execution<br></b> The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.
</p>
<p>
<b>Isolation<br></b> Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.
</p>
<p>
<b>Delivery latency<br></b> 95% of tasks begin execution within five seconds from their scheduled execution time.
</p>
<p>
<b>High availability for task scheduling<br></b> The ATF service is 99.9% available to accept task scheduling requests from any client.
</p>
</div>
<div>
<p id="-lambda-requirements">
<h2>
Lambda requirements
</h2>
</p>
</div>
<div>
<p>
Following are some restrictions we place on the callback logic (lambda):
</p>
<p>
<b>Idempotence</b><br>
A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.
</p>
<p>
<b>Resiliency</b><br>
Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.
</p>
<p>
<b>Terminal state handling<br></b> ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries.&nbsp;
</p>
</div>
<div>
<p id="architecture">
<h2>
Architecture
</h2>
</p>
</div>
<div>
<figure>
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png" aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688" width="1440">
<figcaption>
Async Task Framework (ATF) [Fig 1]
</figcaption>
</figure>
</div>
<div>
<p>
In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.)&nbsp;In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox <a href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc">uses gRPC</a> for remote calls and our in-house <a href="https://dropbox.tech/infrastructure/reintroducing-edgestore">Edgestore</a> to store tasks.
</p>
<p>
ATF consists of the following components:&nbsp;
</p>
<ul>
<li>Frontend
</li>
<li>Task Store
</li>
<li>Store Consumer
</li>
<li>Queue
</li>
<li>Controller
</li>
<li>Executor
</li>
<li>Heartbeat and Status Controller (HSC)<span><br></span>
</li>
</ul>
<p>
<span><b>Frontend</b><br>
This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br>
</p>
<p>
<b>Task Store<br></b> ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be&nbsp;found in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">D</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">ata</a> <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">M</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">odel</a> section below.
</p>
<p>
<b>Store Consumer<br></b> The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system.&nbsp;
</p>
<p>
Below is a simple walkthrough of the Store Consumer’s function:&nbsp;
</p>
</div>
<div>
<pre><code>repeat every second:
1. poll tasks ready for execution from task store
2. push tasks onto the right queues
3. update task statuses</code></pre>
</div>
<div>
<p>
The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task">Lifecycle of a task</a> section below.
</p>
<p>
<b>Queue<br></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/">Simple Queue Service</a> (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <span>&lt;lambda, priority&gt;</span> &nbsp;pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is <span>#lambdas x #priorities</span>.
</p>
<p>
<b>Controller<br></b> Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues.&nbsp;
</p>
<p>
The Controller serves tasks from its process local queue as a response to <span>NextWork</span> RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to <span>NextWork</span> RPCs.
</p>
<p>
<b>Executor<br></b> The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop:
</p>
</div>
<div>
<pre><code>while True:
w = get_next_work()
do_work(w)</code></pre>
</div>
<div>
<p>
Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.
</p>
<p>
<b>Heartbeat and Status Controller (HSC)</b><br>
The HSC serves RPCs for claiming a task for execution (<span>ClaimTask</span>), setting task status after execution (<span>SetResults</span>) and heartbeats during task execution (<span>Heartbeat</span>). <span>ClaimTask</span> requests originate from the Controllers in response to <span>NextWork</span> requests. <span>Heartbeat</span> and <span>SetResults</span> requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.
</p>
</div>
<div>
<p id="data-model">
<h2>
Data model
</h2>
</p>
</div>
<div>
<p>
ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (<span>assoc</span>), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.
</p>
<p>
Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.
</p>
</div>
<div>
<p id="lifecycle-of-a-task">
<h2>
Lifecycle of a task
</h2>
</p>
</div>
<div>
<ol>
<li>Client performs a <span>Schedule</span> RPC call to <b>Frontend</b> with task information, including execution time.&nbsp;
</li>
<li>Frontend creates Edgestore <span>entity</span> and <span>assoc</span> for the task.&nbsp;
</li>
<li>When it is time to process the task, <b>Store Consumer</b> pulls the task from <b>Edgestore</b> and pushes it to a related <b>SQS</b> queue.&nbsp;
</li>
<li>
<b>Executor</b> makes <span>NextWork</span> RPC call to <b>Controller</b>, which pulls tasks from the <b>SQS</b> queue, makes a <span>ClaimTask</span> RPC to the HSC and then returns the task to the <b>Executor</b>.&nbsp;
</li>
<li>
<b>Executor</b> invokes the callback for the task. While processing, <b>Executor</b> performs <span>Heartbeat</span> RPC calls to <b>Heartbeat and Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs <span>TaskStatus</span> RPC call to <b>HSC</b>.&nbsp;
</li>
<li>Upon getting <span>Heartbeat</span> and <span>TaskStatus</span> RPC calls, <b>HSC</b> updates the <b>Edgestore</b> entity and <span>assoc</span>.
</li>
</ol>
<p>
Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the <span>assoc</span>. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.
</p>
<p>
Following are the task entity and association states in ATF and their corresponding timestamp updates:
</p>
<table readabilityDataTable="1">
<tbody>
<tr>
<td>
<p>
<b>Entity status</b>
</p>
</td>
<td>
<p>
<b>Assoc status</b>
</p>
</td>
<td>
<p>
<b>next trigger timestamp in Assoc</b>
</p>
</td>
<td>
<p>
<b>Comment</b>
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>new</span>
</p>
</td>
<td>
<p>
<span>scheduled_timestamp</span> of the task
</p>
</td>
<td>
<p>
Pick up new tasks that are ready.&nbsp;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>enqueued</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>enqueued_timestamp</span> + <span>enqueue_timeout</span>
</p>
</td>
<td>
<p>
Re-enqueue task if it has been in <span>enqueued</span> state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>claimed</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>claimed_timestamp</span> + <span>claim_timeout</span>
</p>
</td>
<td>
<p>
Re-enqueue if task is claimed but never transfered to <span>processing</span>. This can happen if Controller is down after claiming a task. Task status is changed to <span>enqueued</span> after re-enqueue.
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>processing</span>
</p>
</td>
<td>
<p>
<span>started</span>
</p>
</td>
<td>
<p>
<span>heartbeat_timestamp</span> + <span>heartbeat_timeout</span>`
</p>
</td>
<td>
<p>
Re-enqueue if task hasn’t sent <span>heartbeat</span> for too long. This can happen if Executor is down. Task status is changed to <span>enqueued</span> after re-enqueue.&nbsp;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>retriable failure</span>
</p>
</td>
<td>
<p>
started
</p>
</td>
<td>
<p>
compute <span>next_timestamp</span> according to backoff logic
</p>
</td>
<td>
<p>
Exponential backoff for tasks with retriable failure.&nbsp;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span>success</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p>
N/A
</p>
</td>
<td>
</td>
</tr>
<tr>
<td>
<p>
<span>fatal_failure</span>
</p>
</td>
<td>
<p>
<span>completed</span>
</p>
</td>
<td>
<p>
N/A
</p>
</td>
<td>
</td>
</tr>
</tbody>
</table>
<p>
The store consumer polls for tasks based on the following query:
</p>
<p>
<span>assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br></span>
</p>
<p>
Below is the state machine that defines task state transitions:&nbsp;<br>
</p>
</div>
<div>
<figure>
<img src="http://fakehost/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png" aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450" width="1440">
</figure>
</div>
<div>
<p id="-achieving-guarantees">
<h2>
Achieving guarantees
</h2>
</p>
</div>
<div>
<p>
<b>At-least-once task execution<br></b> At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a <span>Success</span> or a <span>FatalFailure</span> state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a <span>RetriableFailure</span> state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.
</p>
<p>
<b>No concurrent task execution<br></b> Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (<span>Claimed</span>) before starting execution. Once the task execution is complete, the task status is updated to one of <span>Success</span>, <span>FatalFailure</span> or <span>RetriableFailure</span>. A task can be claimed only if its existing task state is <span>Enqueued</span> (retried tasks go to the <span>Enqueued</span> state as well once they are re-pushed onto SQS).
</p>
<p>
However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution.&nbsp;<br>
</p>
<p>
To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.
</p>
<p>
<b>Isolation<br></b> Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.
</p>
<p>
<b>Delivery latency<br></b> ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.
</p>
</div>
<div>
<p id="ownership-model">
<h2>
Ownership model
</h2>
</p>
</div>
<p>
ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.
</p>
<div>
<p id="-extending-atf">
<h2>
Extending ATF
</h2>
</p>
</div>
<div>
<p>
As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF.&nbsp;
</p>
<p>
<b>Periodic task execution<br></b> Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.
</p>
<p>
<b>Better support for task chaining<br></b> Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.
</p>
<p>
<b>Dead letter queues for misbehaving tasks<br></b> One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.
</p>
<p>
One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.<br>
</p>
</div>
<div>
<p id="conclusion">
<h2>
Conclusion
</h2>
</p>
</div>
<p>
We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br>
&nbsp;
</p>
</div>

868
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/dropbox-blog/source.html vendored Normal file
View File

@ -0,0 +1,868 @@
<!DOCTYPE html>
<html lang="en" xml:lang="en" data-cms-lang="en-us" xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta charset="utf-8" />
<title>
How we designed Dropbox ATF: an async task framework - Dropbox
</title>
<meta name="data-tags" content="Async,Edgestore,Infrastructure,Task Scheduling" />
<meta name="data-tagTaxonomy" content="Async; Edgestore; Infrastructure; Task Scheduling;" />
<meta name="page-id" content="infrastructure-asynchronous-task-scheduling-at-dropbox" />
<meta name="topic" content="Infrastructure" />
<meta name="publishDate" content="2020-11-11 12:00:00.000-0600" />
<meta name="author" content="Arun Sai Krishnan" />
<link rel="canonical" href="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" />
<link rel="icon" href="https://cfl.dropboxstatic.com/static/images/favicon-vflUeLeeY.ico" type="image/x-icon" />
<meta property="og:url" content="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" />
<meta property="og:type" content="article" />
<meta property="og:title" content="How we designed Dropbox’s ATF - an async task framework" />
<meta property="og:image" content="https://aem.dropbox.com/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-Social.png" />
<meta name="twitter:card" content="summary_large_image" />
<meta content="width=device-width,initial-scale=1.0,user-scalable=no" name="viewport" />
<link rel="alternate" hreflang="en-us" href="https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" /><!-- /* Enable rebrand styles */
<sly data-sly-use.inheritUtil="com.dropbox.aem.common.models.utils.InheritanceUtilUse"
data-sly-test.pageStyle="" /> -->
<link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-common/clientlib-cms-common.757d73acbd22d3e2bf4eeb953c16c4d5.css" type="text/css" />
<link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all.fc2ae6db413129b3901dd5a89e64f347.css" type="text/css" /><!--Knotch Integration should be added in header-->
<script src="https://www.knotch-cdn.com/unit/latest/knotch.min.js" data-account="33c0d4ac-b5bc-4168-a95b-e963ec65974d" async="async"></script>
<link rel="stylesheet" href="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content.22c503f9a8a000fceab6a403af5ce96f.css" type="text/css" />
<style>
<![CDATA[
body.stormcrow-animate{opacity:1;}
]]>
</style>
</head>
<body class="tech-blog-article-page__page stormcrow-animate" data-article-uuid="d4052e45-cbcb-4ebb-b834-eb377ab543e8">
<input type="hidden" id="wcmRunmode" name="wcmRunmode" value="publish,prod" />
<script type="text/javascript">
//<![CDATA[
var attr = "tealium_event$cms".split(",");
var utag_data = {}
attr.forEach(function (item) {
if (item && item.indexOf("$") > -1)
utag_data[item.split("$")[0]] = item.split("$")[1];
})
//]]>
</script>
<script type="text/javascript">
//<![CDATA[
(function (a, b, c, d) {
a = "\/\/tags.tiqcdn.com\/utag\/dropbox\/tech\u002Dblog\/prod\/utag.js";
b = document;
c = 'script';
d = b.createElement(c);
d.src = a;
d.type = 'text/java' + c;
d.async = true;
a = b.getElementsByTagName(c)[0];
a.parentNode.insertBefore(d, a);
})();
//]]>
</script>
<header class="dr-header">
<div class="dr-header__sticky-container">
<section class="dr-header__section dr-flex dr-flex--align-center dr-padding-right-40 dr-padding-left-40 dr-header__sticky-content-container dr-header__sticky-content-container--opened dr-container--surface">
<div class="dr-flex-1">
<a class="dr-link dr-link--no-underline dr-link--no-underline-hover dr-typography-t1" href="https://dropbox.tech/">Dropbox.Tech</a>
</div><button class="dr-header__item dr-nav__menu-toggle-button dr-button dr-hide-from-md dr-flex dr-flex-align-center dr-flex-justify-center" data-dr-tooltip="Menu" data-dr-tooltip-theme="white"><svg viewbox="0 0 28 28" fill="none" xmlns="http://www.w3.org/2000/svg" class="dr-width-100 dr-height-100">
<rect x="0.501831" y="8" width="28" height="2" fill="white"></rect>
<rect x="0.500977" y="18" width="28" height="2" fill="white"></rect></svg></button>
<nav class="dr-show-block-from-md dr-nav__nav">
<button class="dr-button dr-nav__menu-close-button"><svg width="20" height="20" viewbox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M19.2875 2.15983L17.6683 0.566406L9.82597 8.28403L2.33211 0.909344L0.71294 2.50277L8.2068 9.87746L0.666992 17.2974L2.28617 18.8908L9.82597 11.4709L17.7143 19.2337L19.3334 17.6403L11.4451 9.87746L19.2875 2.15983Z" fill="white"></path></svg></button>
<ul class="dr-unstyled-list dr-typography-t2 dr-flex dr-nav__nav-list">
<li class="dr-header__item dr-position-relative dr-header__item--with-subnav dr-header__list-item">
<button class="dr-button dr-button--link dr-header__link--with-subnav">Topics</button>
<ul class="dr-unstyled-list dr-display-none dr-header__subnav dr-position-absolute dr-container--surface dr-padding-top-30 dr-padding-left-40 dr-padding-bottom-20 dr-padding-right-40 dr-font-weight-500">
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/application" class="dr-display-block dr-link dr-link--no-underline dr-container--application dr-link--primary">Application</a>
</li>
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/frontend" class="dr-display-block dr-link dr-link--no-underline dr-container--frontend dr-link--primary">Front End</a>
</li>
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/infrastructure" class="dr-display-block dr-link dr-link--no-underline dr-container--infrastructure dr-link--primary">Infrastructure</a>
</li>
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/machine-learning" class="dr-display-block dr-link dr-link--no-underline dr-container--machine-learning dr-link--primary">Machine Learning</a>
</li>
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/mobile" class="dr-display-block dr-link dr-link--no-underline dr-container--mobile dr-link--primary">Mobile</a>
</li>
<li class="dr-header__list-item dr-header__list-item--subnav">
<a href="https://dropbox.tech/security" class="dr-display-block dr-link dr-link--no-underline dr-container--security dr-link--primary">Security</a>
</li>
</ul>
</li>
<li class="dr-header__item dr-header__list-item">
<a href="https://dropbox.tech/developers" class="dr-link dr-link--no-underline dr-nav__main-category">Developers</a>
</li>
<li class="dr-header__item dr-header__list-item">
<a class="dr-link dr-link--no-underline dr-header__link" href="http://dropbox.com/jobs" target="_blank">Jobs</a>
</li>
</ul>
</nav><button data-dark-mode-switcher="" class="dr-header__item dr-header__dark-mode-switcher dr-button dr-button--link dr-cursor-pointer" data-dr-tooltip="Dark Mode" data-dr-tooltip-theme="white" type="button"><img alt="" height="30" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/button_dark-mode-new.svg" width="30" class="dr-header__mode-image" /></button> <button class="dr-header__item dr-header__search-button dr-button dr-button--link dr-cursor-pointer" data-dr-tooltip="Search" data-dr-tooltip-theme="white" type="button"><img alt="" height="20" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/button_search-new.svg" width="20" /></button> <!--search-result-page-only-->
<!--search-result-page-only-->
<div class="dr-header__search dr-display-none">
<button class="dr-header__search-close-button dr-header__item dr-button dr-button--link dr-cursor-pointer" type="button"><svg width="20" height="20" viewbox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd" clip-rule="evenodd" d="M19.2875 2.15983L17.6683 0.566406L9.82597 8.28403L2.33211 0.909344L0.71294 2.50277L8.2068 9.87746L0.666992 17.2974L2.28617 18.8908L9.82597 11.4709L17.7143 19.2337L19.3334 17.6403L11.4451 9.87746L19.2875 2.15983Z" fill="white"></path></svg></button>
<div class="dr-header__search-form-container">
<form action="https://dropbox.tech/search-results.html" class="dr-header__search-form dr-container__content dr-width-100">
<input autocomplete="off" class="dr-header__search-input dr-typography-t3" name="q" placeholder="Search" required="true" type="text" />
<p class="dr-header__search-hint dr-margin-top-30 dr-margin-bottom-0 dr-typography-t5 dr-display-none">
// Press enter to search
</p>
</form>
</div>
</div>
</section>
</div>
</header>
<div class="dr-article-hero">
<div class="dr-article-hero__background-container dr-container--infrastructure">
<picture class="dr-article-hero__background dr-article-hero__background--regular"><source media="( max-width: 375px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-375x150-light.png" /> <source media="( max-width: 376px ) and ( max-width: 1199px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1024x250-light.png" /> <img class="dr-article-hero__background-image" src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1440x305-light.png" alt="" /></picture> <picture class="dr-article-hero__background dr-article-hero__background--dark"><source media="( max-width: 375px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-375x150-dark.png" /> <source media="( max-width: 376px ) and ( max-width: 1199px )" srcset="/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1024x250-dark.png" /> <img class="dr-article-hero__background-image" src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/header/Infrastructure-ATF-1440x305-dark.png" alt="" /></picture>
</div>
<section class="dr-container__content">
<h1 class="dr-display-inline dr-typography-t15 dr-container--surface dr-article-hero__title">
<span class="dr-article-hero__title-container dr-container--infrastructure">How we designed Dropbox ATF: an async task framework</span>
</h1>
<div class="dr-typography-no-space dr-margin-top-10 dr-margin-md-top-20">
<span class="dr-typography-t5">// By Arun Sai Krishnan • Nov 11, 2020</span>
</div>
</section>
</div>
<div class="dr-article-content">
<div class="dr-article-content__scroll-tracker-container dr-container--infrastructure">
<div class="dr-article-content__scroll-tracker"></div>
</div>
<div class="dr-container__content">
<div class="dr-article-content__content-container dr-padding-md-left-80 dr-padding-md-right-80 dr-typography-t12">
<nav class="dr-article-content__side-nav dr-article-content__side-nav--initial dr-typography-t5">
<ol class="dr-article-content__side-nav-list dr-margin-0">
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#introduction" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Introduction</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#glossary" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Glossary</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#features" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Features</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#system-guarantees" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">System guarantees</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#-lambda-requirements" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Lambda requirements</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#architecture" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Architecture</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#data-model" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Data model</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#lifecycle-of-a-task" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Lifecycle of a task</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#-achieving-guarantees" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Achieving guarantees</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#ownership-model" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Ownership model</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#-extending-atf" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Extending ATF</a>
</li>
<li class="dr-article-content__side-nav-list-item dr-margin-bottom-5">
<a href="#conclusion" class="dr-link dr-link--no-underline dr-article-content__side-nav-link">Conclusion</a>
</li>
</ol>
</nav>
<div class="dr-article-content__content">
<div class="aem-Grid aem-Grid--12 aem-Grid--default--12">
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
I joined Dropbox not long after graduating with a Master’s degree in computer science. Aside from an internship, this was my first big-league engineering job. My team had already begun designing a critical internal service that most of our software would use: It would handle asynchronous computing requests behind the scenes, powering everything from dragging a file into a Dropbox folder to scheduling a marketing campaign.
</p>
<p>
This Asynchronous Task Framework (ATF) would replace multiple bespoke async systems used by different engineering teams. It would reduce redundant development, incompatibilities, and reliance on legacy software. There were no open-source projects or buy-not-build solutions that worked well for our use case and scale, so we had to create our own. ATF is both an important and interesting challenge, though, so we were happy to design, build and deploy our own in-house service.
</p>
<p>
ATF not only had to work well, it had to work well at scale: It would be a foundational building block of Dropbox infrastructure. It would need to handle 10,000 async tasks per second from the start, and be architected for future growth. It would need to support nearly 100 unique async task types from the start, again with room to grow. There were at least two dozen engineering teams that would want to use it for entirely different parts of our codebase, for many products and services.&#160;
</p>
<p>
As any engineer would, we Googled to see what other companies with mega-scale services had done to handle async tasks. We were disappointed to find little material published by engineers who built supersized async services.
</p>
<p>
Now that ATF is deployed and currently serving 9,000 async tasks scheduled per second and in use by 28 engineering teams internally, we’re glad to fill that information gap. We’ve documented Dropbox ATF thoroughly, as a reference and guide for the engineering community seeking their own async solutions.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="introduction">
<h2 class="dr-article-content__section-title">
Introduction
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
Scheduling asynchronous tasks on-demand is a critical capability that powers many features and internal platforms at Dropbox. Async Task Framework (ATF) is the infrastructural system that supports this capability at Dropbox through a callback-based architecture. ATF enables developers to define callbacks, and schedule tasks that execute against these pre-defined callbacks.
</p>
<p>
Since its introduction over a year ago, ATF has gone on to become an important building block in the Dropbox infrastructure, used by nearly 30 internal teams across our codebase. It currently supports 100+ use cases which require either immediate or delayed task scheduling.&#160;
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="glossary">
<h2 class="dr-article-content__section-title">
Glossary
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
Some basic terms repeatedly used in this post, defined as used in the context of this discussion.
</p>
<p>
<b>Lambda:</b> A callback implementing business logic.
</p>
<p>
<span><b>Task:</b> Unit of execution of a lambda. Each asynchronous job scheduled with ATF is a task.</span>
</p>
<p>
<span><b>Collection:</b> A labeled subset of tasks belonging to a lambda. If <span class="dr-code">send email</span> is implemented as a lambda, then <span class="dr-code">password reset email</span> and <span class="dr-code">marketing email</span> would be collections.</span>
</p>
<p>
<span><b>&#160;Priority:</b> Labels defining priority of execution of tasks within a lambda.&#160;</span>
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="features">
<h2 class="dr-article-content__section-title">
Features
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
<b>Task scheduling</b><br />
Clients can schedule tasks to execute at a specified time. Tasks can be scheduled for immediate execution, or delayed to fit the use case.
</p>
<p>
<b>Priority based execution</b><br />
Tasks should be associated with a priority. Tasks with higher priority should get executed before tasks with a lower priority once they are ready for execution.
</p>
<p>
<b>Task gating</b><br />
ATF enables the the gating of tasks based on lambda, or a subset of tasks on a lambda based on collection. Tasks can be gated to be completely dropped or paused until a suitable time for execution.
</p>
<p>
<b>Track task status</b><br />
Clients can query the status of a scheduled task.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="system-guarantees">
<h2 class="dr-article-content__section-title">
System guarantees
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
<b>At-least once task execution<br /></b> The ATF system guarantees that a task is executed at least once after being scheduled. Execution is said to be complete once the user-defined callback signals task completion to the ATF system.
</p>
<p>
<b>No concurrent task execution<br /></b> The ATF system guarantees that at most one instance of a task will be actively executing at any given in point. This helps users write their callbacks without designing for concurrent execution of the same task from different locations.
</p>
<p>
<b>Isolation<br /></b> Tasks in a given lambda are isolated from the tasks in other lambdas. This isolation spans across several dimensions, including worker capacity for task execution and resource use for task scheduling. Tasks on the same lambda but different priority levels are also isolated in their resource use for task scheduling.
</p>
<p>
<b>Delivery latency<br /></b> 95% of tasks begin execution within five seconds from their scheduled execution time.
</p>
<p>
<b>High availability for task scheduling<br /></b> The ATF service is 99.9% available to accept task scheduling requests from any client.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="-lambda-requirements">
<h2 class="dr-article-content__section-title">
Lambda requirements
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
Following are some restrictions we place on the callback logic (lambda):
</p>
<p>
<b>Idempotence</b><br />
A single task on a lambda can be executed multiple times within the ATF system. Developers should ensure that their lambda logic and correctness of task execution in clients are not affected by this.
</p>
<p>
<b>Resiliency</b><br />
Worker processes which execute tasks might die at any point during task execution. ATF retries abruptly interrupted tasks, which could also be retried on different hosts. Lambda owners must design their lambdas such that retries on different hosts do not affect lambda correctness.
</p>
<p>
<b>Terminal state handling<br /></b> ATF retries tasks until they are signaled to be complete from the lambda logic. Client code can mark a task as successfully completed, fatally terminated, or retriable. It is critical that lambda owners design clients to signal task completion appropriately to avoid misbehavior such as infinite retries.&#160;
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="architecture">
<h2 class="dr-article-content__section-title">
Architecture
</h2>
</div>
</div>
<div class="image c04-image aem-GridColumn aem-GridColumn--default--12">
<div class="dr-image image cq-dd-image">
<figure class="dr-margin-0 dr-display-inline-block">
<img src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x844px-1.png" aria-hidden="false" alt="Async Task Framework (ATF) [Fig 1]" height="1688" width="1440" />
<figcaption class="dr-typography-t5 dr-color-ink-60">
Async Task Framework (ATF) [Fig 1]
</figcaption>
</figure>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.)&#160;In this section, we describe the high-level architecture of ATF and give brief description of its different components. (See Fig. 1 above.) Dropbox <a href="https://dropbox.tech/infrastructure/courier-dropbox-migration-to-grpc">uses gRPC</a> for remote calls and our in-house <a href="https://dropbox.tech/infrastructure/reintroducing-edgestore">Edgestore</a> to store tasks.
</p>
<p>
ATF consists of the following components:&#160;
</p>
<ul>
<li>Frontend
</li>
<li>Task Store
</li>
<li>Store Consumer
</li>
<li>Queue
</li>
<li>Controller
</li>
<li>Executor
</li>
<li>Heartbeat and Status Controller (HSC)<span><br /></span>
</li>
</ul>
<p>
<span><b>Frontend</b><br />
This is the service that schedules requests via an RPC interface. The frontend accepts RPC requests from clients and schedules tasks by interacting with ATF’s task store described below.</span><br />
</p>
<p>
<b>Task Store<br /></b> ATF tasks are stored in and triggered from the task store. The task store could be any generic data store with indexed querying capability. In ATF’s case, We use our in-house metadata store Edgestore to power the task store. More details can be&#160;found in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">D</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">ata</a> <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">M</a><a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=395988446153757833740421&amp;h2=Data-model">odel</a> section below.
</p>
<p>
<b>Store Consumer<br /></b> The Store Consumer is a service that periodically polls the task store to find tasks that are ready for execution and pushes them onto the right queues, as described in the queue section below. These could be tasks that are newly ready for execution, or older tasks that are ready for execution again because they either failed in a retriable way on execution, or were dropped elsewhere within the ATF system.&#160;
</p>
<p>
Below is a simple walkthrough of the Store Consumer’s function:&#160;
</p>
</div>
<div class="dr-code-container aem-GridColumn aem-GridColumn--default--12">
<button class="dr-code-container__copy-button dr-button dr-typography-t17">Copy</button>
<pre class="dr-code-container__pre"><code class="dr-code-container__code dr-typography-t5">repeat every second:
1. poll tasks ready for execution from task store
2. push tasks onto the right queues
3. update task statuses</code></pre>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
The Store Consumer polls tasks that failed in earlier execution attempts. This helps with the at-least-once guarantee that the ATF system provides. More details on how the Store Consumer polls new and previously failed tasks is presented in the <a href="https://paper.dropbox.com/doc/How-we-designed-Dropboxs-ATF-an-async-task-framework--A~wmq5aW48OkHns4LzkM~o6zAg-cf95JuxevqilF2iWWATj6#:uid=342792671048375002388848&amp;h2=Lifecycle-of-a-task">Lifecycle of a task</a> section below.
</p>
<p>
<b>Queue<br /></b> ATF uses AWS <a href="https://aws.amazon.com/sqs/" style="background-color: rgb(255,255,255);">Simple Queue Service</a> (SQS) to queue tasks internally. These queues act as a buffer between the Store Consumer and Controllers (described below). Each <span class="dr-code">&lt;lambda, priority&gt;</span> &#160;pair gets a dedicated SQS queue. The total number of SQS queues used by ATF is <span class="dr-code">#lambdas x #priorities</span>.
</p>
<p>
<b>Controller<br /></b> Worker hosts are physical hosts dedicated for task execution. Each worker host has one controller process responsible for polling tasks from SQS queues in a background thread, and then pushing them onto process local buffered queues. The Controller is only aware of the lambdas it is serving and thus polls only the limited set of necessary queues.&#160;
</p>
<p>
The Controller serves tasks from its process local queue as a response to <span class="dr-code">NextWork</span> RPCs. This is the layer where execution level task prioritization occurs. The Controller has different process level queues for tasks of different priorities and can thus prioritize tasks in response to <span class="dr-code">NextWork</span> RPCs.
</p>
<p>
<b>Executor<br /></b> The Executor is a process with multiple threads, responsible for the actual task execution. Each thread within an Executor process follows this simple loop:
</p>
</div>
<div class="dr-code-container aem-GridColumn aem-GridColumn--default--12">
<button class="dr-code-container__copy-button dr-button dr-typography-t17">Copy</button>
<pre class="dr-code-container__pre"><code class="dr-code-container__code dr-typography-t5">while True:
w = get_next_work()
do_work(w)</code></pre>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
Each worker host has a single Controller process and multiple executor processes. Both the Controller and Executors work in a “pull” model, in which active loops continuously long-poll for new work to be done.
</p>
<p>
<b>Heartbeat and Status Controller (HSC)</b><br />
The HSC serves RPCs for claiming a task for execution (<span class="dr-code">ClaimTask</span>), setting task status after execution (<span class="dr-code">SetResults</span>) and heartbeats during task execution (<span class="dr-code">Heartbeat</span>). <span class="dr-code">ClaimTask</span> requests originate from the Controllers in response to <span class="dr-code">NextWork</span> requests. <span class="dr-code">Heartbeat</span> and <span class="dr-code">SetResults</span> requests originate from executor processes during and after task execution. The HSC interacts with the task store to update the task status on the kind of request it receives.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="data-model">
<h2 class="dr-article-content__section-title">
Data model
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
ATF uses our in-house metadata store, Edgestore, as a task store. Edgestore objects can be Entities or Associations (<span class="dr-code">assoc</span>), each of which can have user-defined attributes. Associations are used to represent relationships between entities. Edgestore supports indexing only on attributes of associations.
</p>
<p>
Based on this design, we have two kinds of ATF-related objects in Edgestore. The ATF association stores scheduling information, such as the next scheduled timestamp at which the Store Consumer should poll a given task (either for the first time or for a retry). The ATF entity stores all task related information that is used to track the task state and payload for task execution. We query on associations from the Store Consumer in a pull model to pick up tasks ready for execution.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="lifecycle-of-a-task">
<h2 class="dr-article-content__section-title">
Lifecycle of a task
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<ol>
<li>Client performs a <span class="dr-code">Schedule</span> RPC call to <b>Frontend</b> with task information, including execution time.&#160;
</li>
<li>Frontend creates Edgestore <span class="dr-code">entity</span> and <span class="dr-code">assoc</span> for the task.&#160;
</li>
<li>When it is time to process the task, <b>Store Consumer</b> pulls the task from <b>Edgestore</b> and pushes it to a related <b>SQS</b> queue.&#160;
</li>
<li>
<b>Executor</b> makes <span class="dr-code">NextWork</span> RPC call to <b>Controller</b>, which pulls tasks from the <b>SQS</b> queue, makes a <span class="dr-code">ClaimTask</span> RPC to the HSC and then returns the task to the <b>Executor</b>.&#160;
</li>
<li>
<b>Executor</b> invokes the callback for the task. While processing, <b>Executor</b> performs <span class="dr-code">Heartbeat</span> RPC calls to <b>Heartbeat and Status Controller (HSC)</b>. Once processing is done, <b>Executor</b> performs <span class="dr-code">TaskStatus</span> RPC call to <b>HSC</b>.&#160;
</li>
<li>Upon getting <span class="dr-code">Heartbeat</span> and <span class="dr-code">TaskStatus</span> RPC calls, <b>HSC</b> updates the <b>Edgestore</b> entity and <span class="dr-code">assoc</span>.
</li>
</ol>
<p>
Every state update in the lifecycle of a task is accompanied by an update to the next trigger timestamp in the <span class="dr-code">assoc</span>. This ensures that the Store Consumer pulls the task again if there is no change in state of the task within the next trigger timestamp. This helps ATF achieve its at-least-once delivery guarantee by ensuring that no task is dropped.
</p>
<p>
Following are the task entity and association states in ATF and their corresponding timestamp updates:
</p>
<table>
<tbody>
<tr>
<td>
<p>
<b>Entity status</b>
</p>
</td>
<td>
<p>
<b>Assoc status</b>
</p>
</td>
<td>
<p>
<b>next trigger timestamp in Assoc</b>
</p>
</td>
<td>
<p>
<b>Comment</b>
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">new</span>
</p>
</td>
<td>
<p>
<span class="dr-code">new</span>
</p>
</td>
<td>
<p>
<span class="dr-code">scheduled_timestamp</span> of the task
</p>
</td>
<td>
<p>
Pick up new tasks that are ready.&#160;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">enqueued</span>
</p>
</td>
<td>
<p>
<span class="dr-code">started</span>
</p>
</td>
<td>
<p>
<span class="dr-code">enqueued_timestamp</span> + <span class="dr-code">enqueue_timeout</span>
</p>
</td>
<td>
<p>
Re-enqueue task if it has been in <span class="dr-code">enqueued</span> state for too long. This can happen if the queue loses data or the controller goes down after polling the queue and before the task is claimed.
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">claimed</span>
</p>
</td>
<td>
<p>
<span class="dr-code">started</span>
</p>
</td>
<td>
<p>
<span class="dr-code">claimed_timestamp</span> + <span class="dr-code">claim_timeout</span>
</p>
</td>
<td>
<p>
Re-enqueue if task is claimed but never transfered to <span class="dr-code">processing</span>. This can happen if Controller is down after claiming a task. Task status is changed to <span class="dr-code">enqueued</span> after re-enqueue.
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">processing</span>
</p>
</td>
<td>
<p>
<span class="dr-code">started</span>
</p>
</td>
<td>
<p>
<span class="dr-code">heartbeat_timestamp</span> + <span class="dr-code">heartbeat_timeout</span>`
</p>
</td>
<td>
<p>
Re-enqueue if task hasn’t sent <span class="dr-code">heartbeat</span> for too long. This can happen if Executor is down. Task status is changed to <span class="dr-code">enqueued</span> after re-enqueue.&#160;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">retriable failure</span>
</p>
</td>
<td>
<p>
started
</p>
</td>
<td>
<p>
compute <span class="dr-code">next_timestamp</span> according to backoff logic
</p>
</td>
<td>
<p>
Exponential backoff for tasks with retriable failure.&#160;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">success</span>
</p>
</td>
<td>
<p>
<span class="dr-code">completed</span>
</p>
</td>
<td>
<p>
N/A
</p>
</td>
<td>
<p>
&#160;
</p>
</td>
</tr>
<tr>
<td>
<p>
<span class="dr-code">fatal_failure</span>
</p>
</td>
<td>
<p>
<span class="dr-code">completed</span>
</p>
</td>
<td>
<p>
N/A
</p>
</td>
<td>
<p>
&#160;
</p>
</td>
</tr>
</tbody>
</table>
<p>
The store consumer polls for tasks based on the following query:
</p>
<p>
<span class="dr-code">assoc_status= &amp;&amp; next_timestamp&lt;=time.now()<br /></span>
</p>
<p>
Below is the state machine that defines task state transitions:&#160;<br />
</p>
</div>
<div class="image c04-image aem-GridColumn aem-GridColumn--default--12">
<div class="dr-image image cq-dd-image">
<figure class="dr-margin-0 dr-display-inline-block">
<img src="/cms/content/dam/dropbox/tech-blog/en-us/2020/11/atf/diagrams/Techblog-ATF-720x225px-2.png" aria-hidden="false" alt="Task State Transitions [Fig 2]" height="450" width="1440" />
</figure>
</div>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="-achieving-guarantees">
<h2 class="dr-article-content__section-title">
Achieving guarantees
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
<b>At-least-once task execution<br /></b> At-least-once execution is guaranteed in ATF by retrying a task until it completes execution (which is signaled by a <span class="dr-code">Success</span> or a <span class="dr-code">FatalFailure</span> state). All ATF system errors are implicitly considered retriable failures, and lambda owners have an option of marking tasks with a <span class="dr-code">RetriableFailure</span> state. Tasks might be dropped from the ATF execution pipeline in different parts of the system through transient RPC failures and failures on dependencies like Edgestore or SQS. These transient failures at different parts of the system do not affect the at-least-once guarantee, though, because of the system of timeouts and re-polling from Store Consumer.
</p>
<p>
<b>No concurrent task execution<br /></b> Concurrent task execution is avoided through a combination of two methods in ATF. First, tasks are explicitly claimed through an exclusive task state (<span class="dr-code">Claimed</span>) before starting execution. Once the task execution is complete, the task status is updated to one of <span class="dr-code">Success</span>, <span class="dr-code">FatalFailure</span> or <span class="dr-code">RetriableFailure</span>. A task can be claimed only if its existing task state is <span class="dr-code">Enqueued</span> (retried tasks go to the <span class="dr-code">Enqueued</span> state as well once they are re-pushed onto SQS).
</p>
<p>
However, there might be situations where once a long running task starts execution, its heartbeats might fail repeatedly yet the task execution continues. ATF would retry this task by polling it from the store consumer because the heartbeat timeouts would’ve expired. This task can then be claimed by another worker and lead to concurrent execution.&#160;<br />
</p>
<p>
To avoid this situation, there is a termination logic in the Executor processes whereby an Executor process terminates itself as soon as three consecutive heartbeat calls fail. Each heartbeat timeout is large enough to eclipse three consecutive heartbeat failures. This ensures that the Store Consumer cannot pull such tasks before the termination logic ends them—the second method that helps achieve this guarantee.
</p>
<p>
<b>Isolation<br /></b> Isolation of lambdas is achieved through dedicated worker clusters, dedicated queues, and dedicated per-lambda scheduling quotas. In addition, isolation across different priorities within the same lambda is likewise achieved through dedicated queues and scheduling bandwidth.
</p>
<p>
<b>Delivery latency<br /></b> ATF use cases do not require ultra-low task delivery latencies. Task delivery latencies on the order of a couple of seconds are acceptable. Tasks ready for execution are periodically polled by the Store Consumer and this period of polling largely controls the task delivery latency. Using this as a tuning lever, ATF can achieve different delivery latencies as required. Increasing poll frequency reduces task delivery latency and vice versa. Currently, we have calibrated ATF to poll for ready tasks once every two seconds.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="ownership-model">
<h2 class="dr-article-content__section-title">
Ownership model
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
ATF is designed to be a self-serve framework for developers at Dropbox. The design is very intentional in driving an ownership model where lambda owners own all aspects of their lambdas’ operations. To promote this, all lambda worker clusters are owned by the lambda owners. They have full control over operations on these clusters, including code deployments and capacity management. Each executor process is bound to one lambda. Owners have the option of deploying multiple lambdas on their worker clusters simply by spawning new executor processes on their hosts.
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="-extending-atf">
<h2 class="dr-article-content__section-title">
Extending ATF
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
As described above, ATF provides an infrastructural building block for scheduling asynchronous tasks. With this foundation established, ATF can be extended to support more generic use cases and provide more features as a framework. Following are some examples of what could be built as an extension to ATF.&#160;
</p>
<p>
<b>Periodic task execution<br /></b> Currently, ATF is a system for one-time task scheduling. Building support for periodic task execution as an extension to this framework would be useful in unlocking new capabilities for our clients.
</p>
<p>
<b>Better support for task chaining<br /></b> Currently, it is possible to chain tasks on ATF by scheduling a task onto ATF that then schedules other tasks onto ATF during its execution. Although it is possible to do this in the current ATF setup, visibility and control on this chaining is absent at the framework level. Another natural extension here would be to better support task chaining through framework-level visibility and control, to make this use case a first class concept in the ATF model.
</p>
<p>
<b>Dead letter queues for misbehaving tasks<br /></b> One common source of maintenance overhead we observe on ATF is that some tasks get stuck in infinite retry loops due to occasional bugs in lambda logic. This requires manual intervention from the ATF framework owners in some cases where there are a large number of tasks stuck in such loops, occupying a lot of the scheduling bandwidth in the system. Typical manual actions in response to such a situation include pausing execution of the lambdas with misbehaving tasks, or dropping them outright.
</p>
<p>
One way to reduce this operational overhead and provide an easy interface for lambda owners to recover from such incidents would be to create dead letter queues filled with such misbehaving tasks. The ATF framework could impose a maximum number of retries before tasks are pushed onto the dead letter queue. We could create and expose tools that make it easy to reschedule tasks from the dead letter queue back into the ATF system, once the associated lambda bugs are fixed.<br />
</p>
</div>
<div class="section aem-GridColumn aem-GridColumn--default--12">
<div class="dr-article-content__section" id="conclusion">
<h2 class="dr-article-content__section-title">
Conclusion
</h2>
</div>
</div>
<div class="text parbase aem-GridColumn aem-GridColumn--default--12">
<p>
We hope this post helps engineers elsewhere to develop better async task frameworks of their own. Many thanks to everyone who worked on this project: Anirudh Jayakumar, Deepak Gupta, Dmitry Kopytkov, Koundinya Muppalla, Peng Kang, Rajiv Desai, Ryan Armstrong, Steve Rodrigues, Thomissa Comellas, Xiaonan Zhang and Yuhuan Du.<br />
&#160;
</p>
</div>
</div>
</div>
<hr class="dr-typography-t5 dr-margin-top-50 dr-article-content__divider" />
<div class="dr-typography-t5"></div>
<div class="dr-typography-t5 dr-margin-top-20">
// Tags<br />
<ul class="dr-unstyled-list dr-margin-top-10 dr-typography-t4">
<li class="dr-container--infrastructure dr-display-inline-block dr-margin-right-10 dr-margin-bottom-10">
<a class="dr-link dr-pill dr-pill--primary dr-link--no-underline" href="https://dropbox.tech/infrastructure">Infrastructure</a>
</li>
<li class="dr-display-inline-block dr-margin-right-10">
<a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.task-scheduling">Task Scheduling</a>
</li>
<li class="dr-display-inline-block dr-margin-right-10">
<a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.async">Async</a>
</li>
<li class="dr-display-inline-block dr-margin-right-10">
<a class="dr-link dr-pill dr-link--no-underline" href="https://dropbox.tech/tag-results.edgestore">Edgestore</a>
</li>
</ul>
</div>
<div class="dr-typography-t5 dr-margin-top-20 dr-hide-from-md">
// Copy link<br />
<div class="dr-article-content__social-links-tooltip dr-display-none">
Link copied
</div><button class="dr-button dr-button--link dr-link dr-link--no-underline dr-article-content__copy-link" data-dr-tooltip="Copy link"><img alt="Copy link" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/copy.svg" /></button>
</div>
<div class="dr-article-content__social-links">
<ul class="dr-article-content__social-links-list dr-unstyled-list">
<li class="dr-margin-bottom-20">
<div class="dr-article-content__social-links-tooltip dr-typography-t5 dr-display-none">
Link copied
</div><button class="dr-button dr-display-block dr-link dr-link--no-underline dr-article-content__copy-link dr-button--link" data-dr-tooltip="Copy link" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw"><img alt="Copy link" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/copy.svg" /></button>
</li>
<li class="dr-margin-bottom-20">
<a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__twitter-link" data-dr-tooltip="Share on Twitter" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://twitter.com/intent/tweet/?text=How%20we%20designed%20Dropbox%20ATF%3A%20an%20async%20task%20framework&amp;url=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Twitter" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/twitter.svg" /></a>
</li>
<li class="dr-margin-bottom-20">
<a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__facebook-link" data-dr-tooltip="Share on Facebook" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://facebook.com/sharer/sharer.php?u=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Facebook" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/facebook.svg" /></a>
</li>
<li>
<a class="dr-link dr-display-block dr-link--no-underline dr-article-content__share-link dr-article-content__linkedin-link" data-dr-tooltip="Share on Linkedin" data-dr-tooltip-position="cl" data-dr-tooltip-theme="bw" href="https://www.linkedin.com/shareArticle?mini=true&amp;url=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox&amp;title=How%20we%20designed%20Dropbox%20ATF%3A%20an%20async%20task%20framework&amp;source=https://dropbox.tech/infrastructure/asynchronous-task-scheduling-at-dropbox" target="_blank"><img alt="Share on Linkedin" class="dr-display-block dr-invert-on-theme-dark" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content/resources/linkedin.svg" /></a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="aem-Grid aem-Grid--12 aem-Grid--default--12">
<div class="plain-html c17-plain-html aem-GridColumn aem-GridColumn--default--12">
<div class="knotch_placeholder"></div>
</div>
</div>
<footer class="dr-footer">
<div class="dr-container--surface">
<section class="dr-container__content dr-footer__container">
<div class="dr-newsletter-subscription__succeed dr-display-none dr-typography-t5">
<hr class="dr-newsletter-subscription__form-divider" />
<div class="dr-margin-bottom-30 dr-margin-top-30">
<!--// Thanks for subscribing.-->
<div class="dr-show-block-from-lg">
<img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_desktop.png" title="subscription__success" alt="subscription__success" />
</div>
<div class="dr-show-block-from-md dr-hide-from-lg dr-hide-from-sm">
<img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_tablet.png" title="subscription__success" alt="subscription__success" />
</div>
<div class="dr-show-block-from-sm dr-hide-from-lg dr-hide-from-md">
<img src="/cms/content/dam/dropbox/tech-blog/en-us/subscribe/thanksforsubscribing_mobile.png" title="subscription__success" alt="subscription__success" />
</div>
</div>
<hr class="dr-newsletter-subscription__form-divider" />
</div>
<form role="form" class="dr-typography-t5 dr-newsletter-subscription__form" novalidate="">
<hr class="dr-newsletter-subscription__form-divider" />
<div class="dr-margin-top-30 dr-margin-bottom-30 dr-margin-md-top-10 dr-margin-md-bottom-10">
// Subscribe to email updates by category
</div>
<div class="dr-margin-left-25">
<p class="dr-newsletter-subscription__topic-error dr-display-none dr-color-tangerine">
Select at least one topic
</p><label class="dr-newsletter-subscription__form-label" for="newsletterForm.application"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.application" name="categories[ ]" type="checkbox" value="Application" data-mid="127814" />Application</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.frontend"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.frontend" name="categories[ ]" type="checkbox" value="Front End" data-mid="127842" />Front End</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.infrastructure"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.infrastructure" name="categories[ ]" type="checkbox" value="Infrastructure" data-mid="127826" />Infrastructure</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.machine-learning"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.machine-learning" name="categories[ ]" type="checkbox" value="Machine Learning" data-mid="127830" />Machine Learning</label><br class="dr-show-block-from-md" />
<label class="dr-newsletter-subscription__form-label" for="newsletterForm.mobile"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.mobile" name="categories[ ]" type="checkbox" value="Mobile" data-mid="127834" />Mobile</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.security"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.security" name="categories[ ]" type="checkbox" value="Security" data-mid="127838" />Security</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.developers"><input class="dr-newsletter-subscription__form-checkbox dr-input" id="newsletterForm.developers" name="categories[ ]" type="checkbox" value="Developers" data-mid="129642" />Developers</label> <label class="dr-newsletter-subscription__form-label" for="newsletterForm.all"><input class="dr-newsletter-subscription__form-checkbox dr-newsletter-subscription__form-checkbox--all dr-input" id="newsletterForm.all" type="checkbox" />All</label>
</div>
<p class="dr-newsletter-subscription__error dr-display-none dr-color-tangerine">
Error occurred!<br />
Please try again later
</p>
<p class="dr-newsletter-subscription__email-error dr-display-none dr-color-tangerine">
Enter a valid address
</p>
<div class="dr-newsletter-subscription__email-container dr-margin-bottom-20 dr-margin-top-40 dr-margin-md-top-0">
<div>
// Type your email address
</div><input autocomplete="off" class="dr-newsletter-subscription__form-input dr-flex-1" name="email" type="email" />
<div class="dr-newsletter-subscription__actions-container">
<div class="dr-newsletter-subscription__loading dr-display-none">
Submitting...
</div><button type="submit" disabled="disabled" class="dr-newsletter-subscription__form-submit dr-button dr-typography-t5">Subscribe</button>
</div>
</div>
<hr class="dr-newsletter-subscription__form-divider" />
</form>
<div class="dr-grid dr-grid--md-2">
<div>
<a href="https://dropbox.com" target="_blank" class="dr-margin-bottom-20 dr-display-block"><img alt="Dropbox" height="40" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all/resources/logo_dropbox.svg" width="164" /></a>
</div>
<ul class="dr-footer-links dr-unstyled-list dr-typography-t10 dr-grid dr-grid--2 dr-grid--column-gap-15">
<li>
<a class="dr-link dr-link--no-underline" href="http://dropbox.com/jobs" target="_blank">Jobs</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://medium.com/@Dropbox" target="_blank">Medium</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://www.dropbox.com/privacy" target="_blank">Privacy</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://twitter.com/Dropbox" target="_blank">twitter</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://www.dropbox.com/terms" target="_blank">Terms</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://www.instagram.com/dropbox" target="_blank">Instagram</a>
</li>
<li>
<a class="dr-link dr-link--no-underline" href="https://blog.dropbox.com/" target="_blank">Work In Progress</a>
</li>
</ul>
</div>
</section>
</div>
</footer>
<div id="u04-snapengage-config" data-snapengage-widget-id="d5c1efed-d0ef-4fca-8c7d-faff398ad272" data-proactive-chat="false" style="display:none;"></div>
<script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-common/clientlib-cms-common.7f3cf4624fd698d8bfec572c3c993880.js"></script>
<script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-all.3230e3eaa6e5a90686710bfde829f620.js"></script>
<script type="text/javascript" src="/cms/etc.clientlibs/settings/wcm/designs/dropbox-tech-blog/clientlib-article-content.2c12dd2925c2dcad6bde22d2ff271137.js"></script>
<script type="application/javascript">
<![CDATA[
document.body.classList.remove('stormcrow-animate');
]]>
</script> <noscript></noscript>
</body>
</html>

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/ebb-org/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/ebb-org/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Bradley M. Kuhn (http:\/\/ebb.org\/bkuhn\/)",
"Direction": null,
"Excerpt": "The website of Bradley M. Kuhn, aka Brad, aka bkuhn. This site includes his GPG keys, resume, blog, projects list, software, interviews, speeches and writing.",
"Image": null,
"Title": "On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )",
"SiteName": null
}

56
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/ebb-org/expected.html vendored Normal file
View File

@ -0,0 +1,56 @@
<div id="contentWithSidebar">
<p>
Tuesday 15 October 2019 by Bradley M. Kuhn
</p>
<p>
The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to <a href="https://www.fsf.org/news/richard-m-stallman-resigns">his resignation from the Free Software Foundation (FSF)</a>, to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities.
</p>
<p>
I have been silent the last month because, until two days ago, I was an at-large member of <a href="https://www.fsf.org/about/staff-and-board">FSF's Board of Directors</a>, and a <a href="https://static.fsf.org/nosvn/fsf-amended-bylaws-current.pdf">Voting Member</a> of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis <a href="https://www.fsf.org/news/richard-m-stallman-resigns">come</a> <a href="https://www.fsf.org/news/fsf-and-gnu">directly</a> from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request.
</p>
<p>
The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the <a href="https://sfconservancy.org/about/board/#bkuhn">Board of Directors</a>) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues.
</p>
<p>
I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis.
</p>
<p>
Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — <em>seem</em> to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me.
</p>
<p>
For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed <a href="https://medium.com/@selamjie/remove-richard-stallman-fec6ec210794">Selam G.'s Medium post</a> raised an entirely new issue. <a href="https://web.archive.org/web/20161107050933/https://www.stallman.org/archives/2016-jul-oct.html#31_October_2016_(Down's_syndrome)">In</a> <a href="https://web.archive.org/web/20170202025227/https://www.stallman.org/archives/2016-nov-feb.html#14_December_2016_(Campaign_of_bull-headed_prudery)">fact</a>, <a href="https://web.archive.org/web/20170224174306/https://www.stallman.org/archives/2016-nov-feb.html#23_February_2017_(A_violent_sex_offender)">RMS'</a> <a href="https://web.archive.org/web/20170612074722/http://stallman.org/archives/2017-mar-jun.html#26_May_2017_(Prudish_ignorantism)">views</a> <a href="https://web.archive.org/web/20170616044924/https://www.stallman.org/archives/2017-mar-jun.html#13_June_2017_(Sex_offender_registry)">and</a> <a href="https://web.archive.org/web/20171020041022/http://stallman.org/archives/2017-jul-oct.html#10_October_2017_(Laws_against_having_sex_with_an_animal)">statements</a> <a href="https://web.archive.org/web/20180131020215/https://stallman.org/archives/2017-jul-oct.html#29_October_2017_(Pestering_women)">posted</a> <a href="https://web.archive.org/web/20180104112431/https://www.stallman.org/archives/2017-nov-feb.html#27_November_2017_(Roy_Moore's_relationships)">on</a> <a href="https://web.archive.org/web/20180509120046/https://stallman.org/archives/2018-mar-jun.html#30_April_2018_(UN_peacekeepers_in_South_Sudan)">stallman.org</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#17_July_2018_(The_bullshitter's_flirting)">about</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#21_August_2018_(Age_and_attraction)">sexual</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">morality</a> <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">escalated</a> <a href="https://web.archive.org/web/20181113161736/https://www.stallman.org/archives/2018-sep-dec.html#6_November_2018_(Sex_according_to_porn)">for</a> <a href="https://web.archive.org/web/20190325024048/https://stallman.org/archives/2019-jan-apr.html#14_February_2019_(Respecting_peoples_right_to_say_no)">the</a> <a href="https://www.stallman.org/archives/2019-may-aug.html#11_June_2019_(Stretching_meaning_of_terms)">worse</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#12_June_2019_(Declining_sex_rates)">over</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#30_July_2019_(Al_Franken)">the</a> <a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">last</a> <a href="https://web.archive.org/web/20191011023557/https://stallman.org/archives/2019-jul-oct.html#21_September_2019_(Sex_workers)">few</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">years</a>. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which <acronym title="in my opinion">IMO</acronym> didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma.
</p>
<p>
IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to <acronym title="Free and Open Source Software">FOSS</acronym>. (Indeed, I would not even be commenting on <em>this issue</em> if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with <a href="https://lwn.net/Articles/770966/">restraint, not with impunity</a>.
</p>
<p>
RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality.
</p>
<p>
And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The <a href="https://www.fsf.org/about/">FSF's mission</a> is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission.
</p>
<p>
I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">this “(re-)definition” of sexual assault</a>. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is <strong><a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">not a “frenzy”</a></strong>; it is a global movement by individuals who have been harmed seeking to hold both bad actors <em>and</em> society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the <a href="https://www.gnu.org/philosophy/freedom-or-power.en.html">essay that I co-wrote with RMS</a> and still find <a href="https://www.gnu.org/gnu/manifesto.en.html">many</a> <a href="https://www.gnu.org/philosophy/free-sw.html">of</a> <a href="https://www.gnu.org/philosophy/why-free.html">RMS'</a> <a href="https://www.gnu.org/philosophy/pragmatic.html">other</a> <a href="https://www.gnu.org/philosophy/microsoft-old.html">essays</a> <a href="https://www.gnu.org/philosophy/gpl-american-way.html">compelling</a>, <a href="https://www.gnu.org/licenses/why-not-lgpl.html">important</a>, <a href="https://www.gnu.org/philosophy/stallman-kth.en.html">and</a> <a href="https://www.gnu.org/philosophy/who-does-that-server-really-serve.en.html">relevant</a>.
</p>
<p>
I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission.
</p>
<p>
As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions.
</p>
<p>
Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn.
</p>
</div><p>
<code>#include &lt;std/disclaimer.h&gt;</code><br>
<code>use Standard::Disclaimer;</code><br>
<code>from standard import disclaimer</code><br>
<code>SELECT full_text FROM standard WHERE type = 'disclaimer';</code>
</p><p>
Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either.
</p><p>
ebb <sup>℠</sup> is a service mark of Bradley M. Kuhn.
</p>

485
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/ebb-org/source.html vendored Normal file
View File

@ -0,0 +1,485 @@
<!DOCTYPE html>
<html lang="en-US" xml:lang="en-US" xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type" />
<title>
On Recent Controversial Events - Bradley M. Kuhn ( Brad ) ( bkuhn )
</title>
<meta content="Bradley M. Kuhn (http://ebb.org/bkuhn/)" name="author" />
<link href="http://ebb.org/bkuhn/rss.xml" rel="alternate" title="Whole Website RSS for Bradley M. Kuhn" type="application/rss+xml" />
<link href="http://ebb.org/bkuhn/blog/rss.xml" rel="alternate" title="Blog RSS for Bradley M. Kuhn" type="application/rss+xml" />
<link href="http://ebb.org/bkuhn/blog/rss.xml" rel="alternate" title="Articles / Interviews RSS for Bradley M. Kuhn" type="application/rss+xml" />
<link href="/css/screen.css" rel="stylesheet" type="text/css" />
<link href="mailto:bkuhn@ebb.org" rel="made" />
<meta content="kuhn, brad, bradley, bkuhn, hacker, gnu, free, software, linux, unix, perl, liberal, pgp, gpg, geek, system, administrator, network, free software, developer, new york, NY, computer, open source, hacker, software, freedom, software freedom" http-equiv="Keywords" name="Keywords" />
<meta content="The website of Bradley M. Kuhn, aka Brad, aka bkuhn. This site includes his GPG keys, resume, blog, projects list, software, interviews, speeches and writing." http-equiv="Description" name="Description" />
</head>
<body>
<div id="site">
<div id="sidebar">
<div id="title">
<p>
<a href="/bkuhn">Bradley M. Kuhn</a> <a href="http://ebb.org/bkuhn/rss.xml"><img alt="[RSS of Whole Site]" border="0" src="/images/feed-icon-14x14.png" /></a>
</p>
</div>
<div id="menu">
<ul>
<li>
<a href="/bkuhn/contact">Contact</a>
</li>
<li>
<a href="/bkuhn/blog">Blog</a> &#160; <a href="http://ebb.org/bkuhn/blog/rss.xml"><img alt="[RSS of Blog]" border="0" src="/images/feed-icon-14x14.png" /></a>
</li>
<li>
<a href="http://identi.ca/bkuhn/">Pump.io Social Network</a>
</li><!-- %li= link_to("About", "/bkuhn/about") -->
<!-- %li= link_to("Speeches / Talks", "/bkuhn/speeches/") -->
<li>
<a href="/bkuhn/articles">Interviews / Articles</a> &#160; <a href="http://ebb.org/bkuhn/articles/rss.xml"><img alt="[RSS of Articles]" border="0" src="/images/feed-icon-14x14.png" /></a>
</li>
<li>
<a href="/bkuhn/code/">Software</a>
</li>
<li>
<a href="/bkuhn/resume">Résumé</a>
</li>
</ul>
</div>
<h2>
<a href="/bkuhn/tags">Tag</a> Cloud
</h2>
<ol id="tag-cloud">
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#accounting">accounting</a>
</li>
<li class="tier-4" title="24 posts">
<a href="/bkuhn/tags.html#advocacy">advocacy</a>
</li>
<li class="tier-4" title="16 posts">
<a href="/bkuhn/tags.html#agpl">agpl</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#android">android</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#apache">apache</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#apple">apple</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#apt">apt</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#artistic">artistic</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#asterisk">asterisk</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#automotive">automotive</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#autonomous">autonomous</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#award">award</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#bilski">bilski</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#canonical">canonical</a>
</li>
<li class="tier-3" title="7 posts">
<a href="/bkuhn/tags.html#cla">cla</a>
</li>
<li class="tier-5" title="39 posts">
<a href="/bkuhn/tags.html#community">community</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#compliance">compliance</a>
</li>
<li class="tier-4" title="15 posts">
<a href="/bkuhn/tags.html#conferences">conferences</a>
</li>
<li class="tier-5" title="37 posts">
<a href="/bkuhn/tags.html#conservancy">conservancy</a>
</li>
<li class="tier-3" title="8 posts">
<a href="/bkuhn/tags.html#copyleft">copyleft</a>
</li>
<li class="tier-5" title="56 posts">
<a href="/bkuhn/tags.html#copyright">copyright</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#cow-orking">cow-orking</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#cpp">cpp</a>
</li>
<li class="tier-3" title="9 posts">
<a href="/bkuhn/tags.html#debian">debian</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#denounce">denounce</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#development">development</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#diversity">diversity</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#emacs">emacs</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#encryption">encryption</a>
</li>
<li class="tier-3" title="10 posts">
<a href="/bkuhn/tags.html#enforcement">enforcement</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#exceptions">exceptions</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#faif">faif</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#fdl">fdl</a>
</li>
<li class="tier-4" title="11 posts">
<a href="/bkuhn/tags.html#for-profit">for-profit</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#fosdem">fosdem</a>
</li>
<li class="tier-4" title="13 posts">
<a href="/bkuhn/tags.html#fsf">fsf</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#gcc">gcc</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#git">git</a>
</li>
<li class="tier-4" title="13 posts">
<a href="/bkuhn/tags.html#gnome">gnome</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#gnu">gnu</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#google">google</a>
</li>
<li class="tier-6" title="107 posts">
<a href="/bkuhn/tags.html#gpl">gpl</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#gpl-compatibility">gpl-compatibility</a>
</li>
<li class="tier-5" title="47 posts">
<a href="/bkuhn/tags.html#gpl-enforcement">gpl-enforcement</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#gplv3">gplv3</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#guadec">guadec</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#identica">identica</a>
</li>
<li class="tier-4" title="17 posts">
<a href="/bkuhn/tags.html#infringement">infringement</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#java">java</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#javascript">javascript</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#jvm">jvm</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#launchpad">launchpad</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#ldap">ldap</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#lgpl">lgpl</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#libreoffice">libreoffice</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#libreplanet">libreplanet</a>
</li>
<li class="tier-6" title="72 posts">
<a href="/bkuhn/tags.html#licensing">licensing</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#lindows">lindows</a>
</li>
<li class="tier-4" title="11 posts">
<a href="/bkuhn/tags.html#linux">linux</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#maemo">maemo</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#mail">mail</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#meego">meego</a>
</li>
<li class="tier-4" title="17 posts">
<a href="/bkuhn/tags.html#microsoft">microsoft</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#mobile">mobile</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#moblin">moblin</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#mono">mono</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#motorola">motorola</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#mta">mta</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#mysql">mysql</a>
</li>
<li class="tier-4" title="13 posts">
<a href="/bkuhn/tags.html#net-services">net-services</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#nlp">nlp</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#nokia">nokia</a>
</li>
<li class="tier-4" title="23 posts">
<a href="/bkuhn/tags.html#non-profit">non-profit</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#np-complete">np-complete</a>
</li>
<li class="tier-3" title="7 posts">
<a href="/bkuhn/tags.html#open-core">open-core</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#open-foam">open-foam</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#oracle">oracle</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#parrot">parrot</a>
</li>
<li class="tier-4" title="16 posts">
<a href="/bkuhn/tags.html#patents">patents</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#perl">perl</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#perljvm">perljvm</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#permissive-license">permissive-license</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#piracy">piracy</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#podcast">podcast</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#podjango">podjango</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#poker">poker</a>
</li>
<li class="tier-4" title="14 posts">
<a href="/bkuhn/tags.html#politics">politics</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#postfix">postfix</a>
</li>
<li class="tier-4" title="13 posts">
<a href="/bkuhn/tags.html#proprietary">proprietary</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#qt">qt</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#replicant">replicant</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#requiem">requiem</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#rtlinux">rtlinux</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#scale">SCALE</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#sco">sco</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#scotus">scotus</a>
</li>
<li class="tier-3" title="5 posts">
<a href="/bkuhn/tags.html#security">security</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#sexism">sexism</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#sflc">sflc</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#slicing">slicing</a>
</li>
<li class="tier-3" title="7 posts">
<a href="/bkuhn/tags.html#social-justice">social-justice</a>
</li>
<li class="tier-3" title="6 posts">
<a href="/bkuhn/tags.html#software">software</a>
</li>
<li class="tier-6" title="108 posts">
<a href="/bkuhn/tags.html#software-freedom">software-freedom</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#speeches">speeches</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#stet">stet</a>
</li>
<li class="tier-2" title="4 posts">
<a href="/bkuhn/tags.html#talks">talks</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#tcl">tcl</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#teaching">teaching</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#tech-press">tech-press</a>
</li>
<li class="tier-5" title="35 posts">
<a href="/bkuhn/tags.html#technology">technology</a>
</li>
<li class="tier-2" title="2 posts">
<a href="/bkuhn/tags.html#thesis">thesis</a>
</li>
<li class="tier-3" title="5 posts">
<a href="/bkuhn/tags.html#trademarks">trademarks</a>
</li>
<li class="tier-3" title="10 posts">
<a href="/bkuhn/tags.html#ubuntu">ubuntu</a>
</li>
<li class="tier-1" title="1 post">
<a href="/bkuhn/tags.html#voip">voip</a>
</li>
<li class="tier-2" title="3 posts">
<a href="/bkuhn/tags.html#xen">xen</a>
</li>
</ol>
<h2>
Powered by
</h2><a href="https://gitorious.org/bkuhn/jekyll/source/HEAD:">A Very Old Fork of Jekyll</a> <a href="https://gitorious.org/bkuhn/website/source/HEAD:">"Source Code" for this site</a>
</div>
<div id="contentWithSidebar">
<div id="post">
<h1>
On Recent Controversial Events
</h1>
<p class="topAttributionWithDate">
Tuesday 15 October 2019 by Bradley M. Kuhn
</p>
<p>
The last 33 days have been unprecedentedly difficult for the software freedom community and for me personally. Folks have been emailing, phoning, texting, tagging me on social media (— the last of which has been funny, because all my social media accounts are placeholder accounts). But, just about everyone has urged me to comment on the serious issues that the software freedom community now faces. Until now, I have stayed silent regarding all these current topics: from Richard M. Stallman (RMS)'s public statements, to <a href="https://www.fsf.org/news/richard-m-stallman-resigns">his resignation from the Free Software Foundation (FSF)</a>, to the Epstein scandal and its connection to MIT. I've also avoided generally commenting on software freedom organizational governance during this period. I did this for good reason, which is explained below. However, in this blog post, I now share my primary comments on the matters that seem to currently be of the utmost attention of the Open Source and Free Software communities.
</p>
<p>
I have been silent the last month because, until two days ago, I was an at-large member of <a href="https://www.fsf.org/about/staff-and-board">FSF's Board of Directors</a>, and a <a href="https://static.fsf.org/nosvn/fsf-amended-bylaws-current.pdf">Voting Member</a> of the FSF. As a member of FSF's two leadership bodies, I was abiding by a reasonable request from the FSF management and my duty to the organization. Specifically, the FSF asked that all communication during the crisis <a href="https://www.fsf.org/news/richard-m-stallman-resigns">come</a> <a href="https://www.fsf.org/news/fsf-and-gnu">directly</a> from FSF officers and not from at-large directors and/or Voting Members. Furthermore, the FSF management asked all Directors and Voting Members to remain silent on this entire matter — even on issues only tangentially related to the current situation, and even when speaking in our own capacity (e.g., on our own blogs like this one). The FSF is an important organization, and I take any request from the FSF seriously — so I abided fully with their request.
</p>
<p>
The situation was further complicated because folks at my employer, Software Freedom Conservancy (where I also serve on the <a href="https://sfconservancy.org/about/board/#bkuhn">Board of Directors</a>) had strong opinions about this matter as well. Fortunately, the FSF and Conservancy both had already created clear protocols for what I should do if ever there was a disagreement or divergence of views between Conservancy and FSF. I therefore was recused fully from the planning, drafting, and timing of Conservancy's statement on this matter. I thank my colleagues at the Conservancy for working so carefully to keep me entirely outside the loop on their statement and to diligently assure that it was straight-forward for me to manage any potential organizational disagreements. I also thank those at the FSF who outlined clear protocols (ahead of time, back in March 2019) in case a situation like this ever came up. I also know my colleagues at Conservancy care deeply, as I do, about the health and welfare of the FSF and its mission of fighting for universal software freedom for all. None of us want, nor have, any substantive disagreement over software freedom issues.
</p>
<p>
I take very seriously my duty to the various organizations where I have (or have had) affiliations. More generally, I champion non-profit organizational transparency. Unfortunately, the current crisis left me in a quandary between the overarching goal of community transparency and abiding by FSF management's directives. Now that I've left the FSF Board of Directors, FSF's Voting Membership, and all my FSF volunteer roles (which ends my 22-year uninterrupted affiliation with the FSF), I can now comment on the substantive issues that face not just the FSF, but the Free Software community as a whole, while continuing to adhere to my past duty of acting in FSF's best interest. In other words, my affiliation with the FSF has come to an end for many good and useful reasons. The end to this affiliation allows me to speak directly about the core issues at the heart of the community's current crisis.
</p>
<p>
Firstly, all these events — from RMS' public comments on the MIT mailing list, to RMS' resignation from the FSF to RMS' discussions about the next steps for the GNU project — <em>seem</em> to many to have happened ridiculously quickly. But it wasn't actually fast at all. In fact, these events were culmination of issues that were slowly growing in concern to many people, including me.
</p>
<p>
For the last two years, I had been a loud internal voice in the FSF leadership regarding RMS' Free-Software-unrelated public statements; I felt strongly that it was in the best interest of the FSF to actively seek to limit such statements, and that it was my duty to FSF to speak out about this within the organization. Those who only learned of this story in the last month (understandably) believed <a href="https://medium.com/@selamjie/remove-richard-stallman-fec6ec210794">Selam G.'s Medium post</a> raised an entirely new issue. <a href="https://web.archive.org/web/20161107050933/https://www.stallman.org/archives/2016-jul-oct.html#31_October_2016_(Down's_syndrome)">In</a> <a href="https://web.archive.org/web/20170202025227/https://www.stallman.org/archives/2016-nov-feb.html#14_December_2016_(Campaign_of_bull-headed_prudery)">fact</a>, <a href="https://web.archive.org/web/20170224174306/https://www.stallman.org/archives/2016-nov-feb.html#23_February_2017_(A_violent_sex_offender)">RMS'</a> <a href="https://web.archive.org/web/20170612074722/http://stallman.org/archives/2017-mar-jun.html#26_May_2017_(Prudish_ignorantism)">views</a> <a href="https://web.archive.org/web/20170616044924/https://www.stallman.org/archives/2017-mar-jun.html#13_June_2017_(Sex_offender_registry)">and</a> <a href="https://web.archive.org/web/20171020041022/http://stallman.org/archives/2017-jul-oct.html#10_October_2017_(Laws_against_having_sex_with_an_animal)">statements</a> <a href="https://web.archive.org/web/20180131020215/https://stallman.org/archives/2017-jul-oct.html#29_October_2017_(Pestering_women)">posted</a> <a href="https://web.archive.org/web/20180104112431/https://www.stallman.org/archives/2017-nov-feb.html#27_November_2017_(Roy_Moore's_relationships)">on</a> <a href="https://web.archive.org/web/20180509120046/https://stallman.org/archives/2018-mar-jun.html#30_April_2018_(UN_peacekeepers_in_South_Sudan)">stallman.org</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#17_July_2018_(The_bullshitter's_flirting)">about</a> <a href="https://web.archive.org/web/20180911075211/https://www.stallman.org/archives/2018-jul-oct.html#21_August_2018_(Age_and_attraction)">sexual</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">morality</a> <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">escalated</a> <a href="https://web.archive.org/web/20181113161736/https://www.stallman.org/archives/2018-sep-dec.html#6_November_2018_(Sex_according_to_porn)">for</a> <a href="https://web.archive.org/web/20190325024048/https://stallman.org/archives/2019-jan-apr.html#14_February_2019_(Respecting_peoples_right_to_say_no)">the</a> <a href="https://www.stallman.org/archives/2019-may-aug.html#11_June_2019_(Stretching_meaning_of_terms)">worse</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#12_June_2019_(Declining_sex_rates)">over</a> <a href="https://web.archive.org/web/20190801201704/https://stallman.org/archives/2019-may-aug.html#30_July_2019_(Al_Franken)">the</a> <a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">last</a> <a href="https://web.archive.org/web/20191011023557/https://stallman.org/archives/2019-jul-oct.html#21_September_2019_(Sex_workers)">few</a> <a href="https://web.archive.org/web/20180924231708/https://stallman.org/archives/2018-jul-oct.html#23_September_2018_(Cody_Wilson)">years</a>. When the escalation started, I still considered RMS both a friend and colleague, and I attempted to argue with him at length to convince him that some of his positions were harmful to sexual assault survivors and those who are sex trafficked, and to the people who devote their lives in service to such individuals. More importantly to the FSF, I attempted to persuade RMS that launching a controversial campaign on sexual behavior and morality was counter to his and FSF's mission to advance software freedom, and told RMS that my duty as an FSF Director was to assure the best outcome for the FSF, which <acronym title="in my opinion">IMO</acronym> didn't include having a leader who made such statements. Not only is human sexual behavior not a topic on which RMS has adequate academic expertise, but also his positions appear to ignore significant research and widely available information on the subject. Many of his comments, while occasionally politically intriguing, lack empathy for people who experienced trauma.
</p>
<p>
IMO, this is not and has never been a Free Speech issue. I do believe freedom of speech links directly to software freedom: indeed, I see the freedom to publish software under Free licenses as almost a corollary to the freedom of speech. However, we do not need to follow leadership from those whose views we fundamentally disagree. Moreover, organizations need not and should not elevate spokespeople and leaders who speak regularly on unrelated issues that organizations find do not advance their mission, and/or that alienate important constituents. I, like many other software freedom leaders, curtail my public comments on issues not related to <acronym title="Free and Open Source Software">FOSS</acronym>. (Indeed, I would not even be commenting on <em>this issue</em> if it had not become a central issue of concern to the software freedom community.) Leaders have power, and they must exercise the power of their words with <a href="https://lwn.net/Articles/770966/">restraint, not with impunity</a>.
</p>
<p>
RMS has consistently argued that there was a campaign of “prudish intimidation” — seeking to keep him quiet about his views on sexuality. After years of conversing with RMS about how his non-software-freedom views were a distraction, an indulgence, and downright problematic, his general response was to make even more public comments of this nature. The issue is not about RMS' right to say what he believes, nor is it even about whether or not you agree or disagree with RMS' statements. The question is whether an organization should have a designated leader who is on a sustained, public campaign advocating about an unrelated issue that many consider controversial. It really doesn't matter what your view about the controversial issue is; a leader who refuses to stop talking loudly about unrelated issues eventually creates an untenable distraction from the radical activism you're actively trying to advance. The message of universal software freedom is a radical cause; it's basically impossible for one individual to effectively push forward two unrelated controversial agendas at once. In short, the radical message of software freedom became overshadowed by RMS' radical views about sexual morality.
</p>
<p>
And here is where I say the thing that may infuriate many but it's what I believe: I think RMS took a useful step by resigning some of his leadership roles at the FSF. I thank RMS for taking that step, and I wish the FSF Directors well in their efforts to assure that the FSF becomes a welcoming organization to all who care about universal software freedom. The <a href="https://www.fsf.org/about/">FSF's mission</a> is essential to our technological future, and we should all support that mission. I care deeply about that mission myself and have worked and will continue to work in our community in the best interest of the mission.
</p>
<p>
I'm admittedly struggling to find a way to work again with RMS, given his views on sexual morality and his behaviors stemming from those views. I explicitly do not agree with <a href="https://web.archive.org/web/20180919100154/https://stallman.org/antiglossary.html#assult">this “(re-)definition” of sexual assault</a>. Furthermore, I believe uninformed statements about sexual assault are irresponsible and cause harm to victims. #MeToo is <strong><a href="https://web.archive.org/web/20190903050208/https://stallman.org/archives/2019-jul-oct.html#27_August_2019_(Me-too_frenzy)">not a “frenzy”</a></strong>; it is a global movement by individuals who have been harmed seeking to hold both bad actors <em>and</em> society-at-large accountable for ignoring systemic wrongs. Nevertheless, I still am proud of the <a href="https://www.gnu.org/philosophy/freedom-or-power.en.html">essay that I co-wrote with RMS</a> and still find <a href="https://www.gnu.org/gnu/manifesto.en.html">many</a> <a href="https://www.gnu.org/philosophy/free-sw.html">of</a> <a href="https://www.gnu.org/philosophy/why-free.html">RMS'</a> <a href="https://www.gnu.org/philosophy/pragmatic.html">other</a> <a href="https://www.gnu.org/philosophy/microsoft-old.html">essays</a> <a href="https://www.gnu.org/philosophy/gpl-american-way.html">compelling</a>, <a href="https://www.gnu.org/licenses/why-not-lgpl.html">important</a>, <a href="https://www.gnu.org/philosophy/stallman-kth.en.html">and</a> <a href="https://www.gnu.org/philosophy/who-does-that-server-really-serve.en.html">relevant</a>.
</p>
<p>
I want the FSF to succeed in its mission and enter a new era of accomplishments. I've spent the last 22 years, without a break, dedicating substantial time, effort, care and loyalty to the various FSF roles that I've had: including employee, volunteer, at-large Director, and Voting Member. Even though my duties to the FSF are done, and my relationship with the FSF is no longer formal, I still think the FSF is a valuable institution worth helping and saving, specifically because the FSF was founded for a mission that I deeply support. And we should also realize that RMS — a human being (who is flawed like the rest of us) — invented that mission.
</p>
<p>
As culture change becomes more rapid, I hope we can find reasonable nuance and moderation on our complex analysis about people and their disparate views, while we also hold individuals fully accountable for their actions. That's the difficulty we face in the post-post-modern culture of the early twenty-first century. Most importantly, I believe we must find a way to stand firm for software freedom while also making a safe environment for victims of sexual assault, sexual abuse, gaslighting, and other deplorable actions.
</p>
<p class="bottomAttributionWithDate">
Posted on Tuesday 15 October 2019 at 09:11 by Bradley M. Kuhn.
</p>
<p class="comments">
Submit comments on this post to <a href="mailto:bkuhn@ebb.org">&lt;bkuhn@ebb.org&gt;</a>.
</p>
<div id="prevlink">
<a class="previous" href="/bkuhn/blog/2019/05/23/github-sponsors.html">← <strong>Previous</strong>: Chasing Quick Fixes To Sustainability</a>
</div>
</div>
</div>
<p id="footer"></p>
<hr />
<br />
<p>
<a href="http://creativecommons.org/licenses/by-sa/3.0/us/" rel="license"><img alt="Creative Commons License" src="http://i.creativecommons.org/l/by-sa/3.0/us/88x31.png" style="border-width:10" /></a> This website and all documents on it are licensed under a <a href="http://creativecommons.org/licenses/by-sa/3.0/us/" rel="license">Creative Commons Attribution-Share Alike 3.0 United States License</a> .
</p>
<hr />
<p>
<code>#include &lt;std/disclaimer.h&gt;</code><br />
<code>use Standard::Disclaimer;</code><br />
<code>from standard import disclaimer</code><br />
<code>SELECT full_text FROM standard WHERE type = 'disclaimer';</code>
</p>
<p>
Both previously and presently, I have been employed by and/or done work for various organizations that also have views on Free, Libre, and Open Source Software. As should be blatantly obvious, this is my website, not theirs, so please do not assume views and opinions here belong to any such organization. Since I do co-own ebb.org with my wife, it may not be so obvious that these aren't her views and opinions, either.
</p>
<p align="right">
— bkuhn
</p>
<hr />
<p>
ebb <sup>℠</sup> is a service mark of Bradley M. Kuhn.
</p>
<address>
<a href="http://ebb.org/bkuhn/">Bradley M. Kuhn</a> <a href="mailto:bkuhn@ebb.org">&lt;bkuhn@ebb.org&gt;</a>
</address>
</div>
</body>
</html>

3
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/folha/expected-images.json vendored Normal file
View File

@ -0,0 +1,3 @@
[
"https:\/\/f.i.uol.com.br\/fotografia\/2018\/12\/21\/15454034955c1cfc67131dc_1545403495_3x2_rt.jpg"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/folha/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Bruno (Henrique Zecchin) Rodrigues",
"Direction": null,
"Excerpt": "Na ocasi\u00e3o, t\u00e9cnico do Corinthians entregou r\u00e9plica do trof\u00e9u ao ex-presidente",
"Image": "https:\/\/f.i.uol.com.br\/fotografia\/2018\/12\/21\/15454034955c1cfc67131dc_1545403495_3x2_rt.jpg",
"Title": "Tite diz que errou ao levar ta\u00e7a da Libertadores a Lula em 2012",
"SiteName": "Folha de S.Paulo"
}

24
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/folha/expected.html vendored Normal file
View File

@ -0,0 +1,24 @@
<div data-share-text data-news-content-text data-disable-copy data-continue-reading data-continue-reading-hide-others=".js-continue-reading-hidden" itemprop="articleBody">
<p>
Após rechaçar <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">um encontro da seleção brasileira com o presidente eleito Jair</a> <a href="https://www1.folha.uol.com.br/esporte/2018/12/tite-se-recusa-a-encontrar-bolsonaro-antes-da-disputa-da-copa-america.shtml">Bolsonaro</a>, o técnico Tite declarou que errou ao levar a taça da Copa Libertadores de 2012, conquistada pelo Corinthians, ao ex-presidente Luiz Inácio Lula da Silva.
</p>
<p>
Ao lado de representantes do clube paulista, o atual comandante do Brasil ainda entregou uma réplica do troféu a Lula.
</p>
<p>
"Em 2012 eu errei. Ele não era presidente, mas fui ao Instituto e mandei felicitações por um aniversário. Não me posicionei politicamente. Não tenho partido político, tenho sim a torcida para que o Brasil seja melhor em igualdade social. E que nossas prioridades sejam educação e punição. Que seja dada a possibilidade de estudo ao garoto de São Braz, que não tem chão batido para ir à escola, ou da periferia de Caixas ou do morro do Rio de Janeiro. Seja dada a ele a prioridade de estudo e não a outras situações", falou Tite ao programa "Grande Círculo", que ainda irá ao ar no SporTV.
</p>
<p>
Na ocasião, Tite e outros representantes do Corinthians <a href="https://www1.folha.uol.com.br/poder/1124743-corinthians-leva-a-taca-da-libertadores-para-lula.shtml">foram ao Instituto Lula para mostrar a taça</a> original da Libertadores ao ex-presidente.
</p>
<p>
O assunto foi levantado&nbsp;porque recentemente Tite foi questionado se aceitaria um encontro da seleção brasileira com Bolsonaro em uma conquista de título ou <a href="https://www1.folha.uol.com.br/esporte/2018/12/selecao-brasileira-jogara-duas-vezes-em-sao-paulo-na-copa-america.shtml">antes da Copa América de 2019</a>, por exemplo. O treinador deixou claro que preferiria evitar esse tipo de formalidade.
</p>
<p>
Apesar disso, Tite não questionou a ação de Palmeiras e CBF, que <a href="https://www1.folha.uol.com.br/esporte/2018/12/cbf-usa-festa-do-palmeiras-para-se-aproximar-de-governo-bolsonaro.shtml">convidaram Bolsonaro para a festa do título do Campeonato Brasileiro</a>. O presidente eleito até levantou a taça conquistada pelo clube alviverde.
</p>
<p>
"Em 2012 eu fiz e errei. O protocolo e a situação gerada no jogo do Palmeiras são fatos de opinião pessoal. CBF e Palmeiras, enquanto instituições têm a opinião. Errei lá atrás, não faria com o presidente antes da Copa e nem agora porque entendo que misturar esporte e política não é legal. Fiz errado lá atrás? Sim. Faria de novo? Não", acrescentou o comandante.
</p>
</div>

2518
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/folha/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/google-sre-book-1/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/google-sre-book-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Written by Rob Ewaschuk\n Edited by Betsy Beyer",
"Direction": null,
"Excerpt": "Google\u2019s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren\u2019t serious enough to trigger a page.",
"Image": null,
"Title": "Google - Site Reliability Engineering",
"SiteName": null
}

458
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/google-sre-book-1/expected.html vendored Normal file
View File

@ -0,0 +1,458 @@
<section data-type="chapter" id="maia-main" role="main">
<h2>
Monitoring Distributed Systems
</h2>
<p>
Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.
</p>
<section data-type="sect1" id="definitions-2ksZhN">
<h2>
Definitions
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="terminology" id="id-DnC1SWFMhD"></a>There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.
</p>
<dl>
<dd>
<p>
Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="white-box monitoring" id="id-9nCjSDS4tZILhX"></a>Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCxSrSgTWIdhb"></a>Testing externally visible behavior as a user would see it.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="defined" id="id-VMCPS2SribIkh4"></a>An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="alerts" data-secondary="defined" id="id-wqC7SvSPUAIVhQ"></a>A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as <em>tickets</em>, <em>email alerts</em>,<sup><a data-type="noteref" id="id-LvQuvtYS7UvI8h4-marker" href="#id-LvQuvtYS7UvI8h4">22</a></sup> and <em>pages</em>.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="root cause" data-secondary="defined" id="id-PnCpSaSKsgIjho"></a>A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, <em>and</em> insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.
</p>
</dd>
<dt id="node-and-machine">
Node and machine
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="machines" data-secondary="defined" id="id-XmC9SkSlfnI1hK"></a>Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple <em>services</em> worth monitoring on a single machine. The services may either be:
</p>
<ul>
<li>Related to each other: for example, a caching server and a web server
</li>
<li>Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like <a href="https://puppetlabs.com/puppet/puppet-open-source" target="_blank">Puppet</a> or <a href="https://www.chef.io/chef/" target="_blank">Chef</a>
</li>
</ul>
</dd>
<dd>
<p>
Any change to a service’s running software or its configuration.
</p>
</dd>
</dl>
</section>
<section data-type="sect1" id="why-monitor-pWsBTZ">
<h2>
Why Monitor?
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="benefits of monitoring" id="id-kVCkSpFnTl"></a>There are many reasons to monitor a system, including:
</p>
<dl>
<dd>
<p>
How big is my database and how fast is it growing? How quickly is my daily-active user count growing?
</p>
</dd>
<dd>
<p>
Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?
</p>
</dd>
<dd>
<p>
Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="benefits of" id="id-rjCXSOS0iDIGT8"></a>Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in <a data-type="xref" href="#xref_monitoring_golden-signals">The Four Golden Signals</a>).
</p>
</dd>
<dd>
<p>
Our latency just shot up; what else happened around the same time?
</p>
</dd>
</dl>
<p>
System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.
</p>
<p>
Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."
</p>
<p>
Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.
</p>
</section>
<section data-type="sect1" id="setting-reasonable-expectations-for-monitoring-o8svcM">
<h2>
Setting Reasonable Expectations for Monitoring
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="setting expectations for" id="id-4nCqSYFQcE"></a>Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)
</p>
<p>
<a data-type="indexterm" data-primary="post hoc analysis" id="id-JnCDSjIVcG"></a>In general, Google has trended toward simpler and faster monitoring systems, with better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.
</p>
<p>
<a data-type="indexterm" data-primary="dependency hierarchies" id="id-9nCjSOtmcj"></a>Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.
</p>
<p>
Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.
</p>
<p>
Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.
</p>
</section>
<section data-type="sect1" id="symptoms-versus-causes-g0sEi4">
<h2>
Symptoms Versus Causes
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="symptoms vs. causes" id="id-JnCDSlFmiG"></a>Your monitoring system should address two questions: what’s broken, and why?
</p>
<p>
The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table 6-1</a> lists some hypothetical symptoms and corresponding causes.
</p>
<table id="table_monitoring_symptoms" readabilityDataTable="1">
<caption>
<span>Table 6-1.</span> Example symptoms and causes
</caption>
<thead>
<tr>
<th>
<strong>Symptom</strong>
</th>
<th>
<strong>Cause</strong>
</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<p>
<strong>I’m serving HTTP 500s or 404s</strong>
</p>
</td>
<td>
<p>
Database servers are refusing connections
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>My responses are slow</strong>
</p>
</td>
<td>
<p>
CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
</p>
</td>
<td>
<p>
Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Private content is world-readable</strong>
</p>
</td>
<td>
<p>
A new software push caused ACLs to be forgotten and allowed all requests
</p>
</td>
</tr>
</tbody>
</table>
<p>
"What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.
</p>
</section>
<section data-type="sect1" id="black-box-versus-white-box-q8sJuw">
<h2>
Black-Box Versus White-Box
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="blackbox vs. whitebox" id="id-9nCjSvFVuj"></a><a data-type="indexterm" data-primary="white-box monitoring" id="id-ZbC1FMFEu7"></a><a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCXIGFvuy"></a>We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.
</p>
<p>
Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.
</p>
<p>
When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.
</p>
<p>
For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.
</p>
</section>
<section data-type="sect1" id="xref_monitoring_golden-signals">
<h2>
The Four Golden Signals
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="four golden signals of" id="id-ZbCxSMFjU7"></a>The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.
</p>
<dl>
<dd>
<p>
<a data-type="indexterm" data-primary="service latency" data-secondary="monitoring for" id="id-yYCASJS9FKIWUb"></a><a data-type="indexterm" data-primary="latency" data-secondary="monitoring for" id="id-VMCpF2SXFbIwU4"></a><a data-type="indexterm" data-primary="request latency" id="id-rjCeIOSKFDIaU8"></a><a data-type="indexterm" data-primary="user requests" data-secondary="request latency monitoring" id="id-wqCDtvSGFAIMUQ"></a>The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="user requests" data-secondary="traffic analysis" id="id-rjCXSOSxtDIaU8"></a><a data-type="indexterm" data-primary="traffic analysis" id="id-wqC4FvSBtAIMUQ"></a>A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per <span>second</span>.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="error rates" id="id-x1C4SjSlTLIMUJ"></a><a data-type="indexterm" data-primary="user requests" data-secondary="monitoring failures" id="id-PnCxFaS0TgIVUo"></a>The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.
</p>
</dd>
<dd>
<p>
<a data-type="indexterm" data-primary="saturation" id="id-OnCNS2S4iDIYU8"></a>How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.
</p>
<p>
In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.
</p>
<p>
Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."
</p>
</dd>
</dl>
<p>
If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.
</p>
</section>
<section data-type="sect1" id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
<h2>
Worrying About Your Tail (or, Instrumentation and Performance)
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="instrumentation and performance" id="id-zdCxSGFQCy"></a><a data-type="indexterm" data-primary="performance" data-secondary="monitoring" id="id-yYCyFpFdCr"></a>When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.<sup><a data-type="noteref" id="id-QQLuAIXFxCz-marker" href="#id-QQLuAIXFxCz">23</a></sup> If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your <span>frontend</span>.
</p>
<p>
The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.
</p>
</section>
<section data-type="sect1" id="choosing-an-appropriate-resolution-for-measurements-vJsBsE">
<h2>
Choosing an Appropriate Resolution for Measurements
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="resolution" id="id-yYCASpFxsr"></a>Different aspects of a system should be measured with different levels of granularity. For example:
</p>
<ul>
<li>Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
</li>
<li>On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
</li>
<li>Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.
</li>
</ul>
<p>
Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:
</p>
<ol>
<li>Record the current CPU utilization each second.
</li>
<li>Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
</li>
<li>Aggregate those values every minute.
</li>
</ol>
<p>
This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.
</p>
</section>
<section data-type="sect1" id="as-simple-as-possible-no-simpler-lqskHx">
<h2>
As Simple as Possible, No Simpler
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="avoiding complexity in" id="id-VMCPSrFpHm"></a>Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:
</p>
<ul>
<li>Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
</li>
<li>Extra code to detect and expose possible causes
</li>
<li>Associated dashboards for each of these possible causes
</li>
</ul>
<p>
The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.
</p>
<p>
Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:
</p>
<ul>
<li>The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
</li>
<li>Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
</li>
<li>Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.
</li>
</ul>
<p>
In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).
</p>
</section>
<section data-type="sect1" id="tying-these-principles-together-nqsJfw">
<h2>
Tying These Principles Together
</h2>
<p>
The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.
</p>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="creating rules for" id="id-wqC7SDIvfj"></a>When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:<sup><a data-type="noteref" id="id-a82udF8IBfx-marker" href="#id-a82udF8IBfx">24</a></sup>
</p>
<ul>
<li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent, actionable, and actively or imminently user-visible?<sup><a data-type="noteref" id="id-0vYuEFpSjSMtLfG-marker" href="#id-0vYuEFpSjSMtLfG">25</a></sup>
</li>
<li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
</li>
<li>Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
</li>
<li>Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
</li>
<li>Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?
</li>
</ul>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="monitoring philosophy" id="id-PnCpSwhJfa"></a>These questions reflect a fundamental philosophy on pages and pagers:
</p>
<ul>
<li>Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
</li>
<li>Every page should be actionable.
</li>
<li>Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
</li>
<li>Pages should be about a novel problem or an event that hasn’t been seen before.
</li>
</ul>
<p>
Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.
</p>
</section>
<section data-type="sect1" id="monitoring-for-the-long-term-NbsNS8">
<h2>
Monitoring for the Long Term
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="challenges of" id="id-wqC7SPFMSj"></a>In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.
</p>
<p>
It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.
</p>
<section data-type="sect2" id="bigtable-sre-a-tale-of-over-alerting-dbsXtjSM">
<p>
<a data-type="indexterm" id="MDSbig6" data-primary="monitoring distributed systems" data-secondary="case studies"></a><a data-type="indexterm" data-primary="Bigtable" id="id-XmCpFOFytySv"></a>Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see <a data-type="xref" href="http://fakehost/sre/sre-book/chapters/service-level-objectives">Service Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.
</p>
<p>
Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.
</p>
<p>
To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.
</p>
<p>
This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.
</p>
</section>
<section data-type="sect2" id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
<p>
<a data-type="indexterm" data-primary="Gmail" id="id-XmC9SOFZhySv"></a>In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.
</p>
<p>
At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.
</p>
<p>
To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.
</p>
<p>
This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.
</p>
<p>
Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.<a data-type="indexterm" data-primary data-startref="MDSbig6" id="id-oPCASqT2hLSk"></a>
</p>
</section>
<section data-type="sect2" id="the-long-run-MQsWTMS7">
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="short- vs. long-term availability" id="id-jyCxSoFETNSd"></a>A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall <em>level</em> of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.
</p>
</section>
</section>
<section data-type="sect1" id="conclusion-8ksvFj">
<h2>
Conclusion
</h2>
<p>
A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.
</p>
<p>
Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.
</p>
</section>
</section>

742
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/google-sre-book-1/source.html vendored Normal file
View File

@ -0,0 +1,742 @@
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
<head>
<meta charset="utf-8" />
<meta content="initial-scale=1, minimum-scale=1, width=device-width" name="viewport" />
<title>
Google - Site Reliability Engineering
</title>
<meta name="referrer" content="no-referrer" />
<link rel="apple-touch-icon-precomposed" sizes="180x180" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Google+Sans:400|Roboto:400,400italic,500,500italic,700,700italic|Roboto+Mono:400,500,700|Material+Icons" />
<link rel="icon" type="image/png" sizes="32x32" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
<link rel="icon" type="image/png" sizes="16x16" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
<link rel="shortcut icon" href="https://lh3.googleusercontent.com/Yf2DCX8RKda6r4Jml9DLMByS2zQCBFs3kQpvBfN8UgIh4YVWIYSYIQOoTxJriyuM26cT5PDjyEb5aynDQ0Xyz46yHKnfg8JlUbDW" />
<link href="/sre/sre-book/static/css/index.min.css?cache=0ffc48d" rel="stylesheet" />
<script>
<![CDATA[
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
ga('create', 'UA-75468017-1', 'auto');
ga('send', 'pageview');
]]>
</script>
<script src="/sre/sre-book/static/js/detect.min.js?cache=4cb778b"></script>
</head>
<body>
<main>
<div ng-controller="HeaderCtrl as headerCtrl">
<div id="curtain" class="menu-closed"></div>
<div class="header clearfix">
<a id="burger-menu" class="expand"></a>
<h2 class="chapter-title">
Chapter 6 - Monitoring Distributed Systems
</h2>
</div>
<div id="overlay-element" class="expands">
<div class="logo">
<a href="https://www.google.com"><img src="https://lh3.googleusercontent.com/YoVRtLOHMSRYQZ3OhFL8RIamcjFYbmQXX4oAQx02MRqqY9zlKNvsuZpS73khXiOqTH3qrFW27VrERJJIHTjPk-tAh46q8-Fd4w6qlw" alt="Google" /></a>
</div>
<ol id="drop-down" class="dropdown-content hide">
<li>
<a class="menu-buttons" href="/sre/sre-book/toc/">Table of Contents</a>
</li>
<li>
<a href="/sre/sre-book/chapters/foreword" class="menu-buttons">Foreword</a>
</li>
<li>
<a href="/sre/sre-book/chapters/preface" class="menu-buttons">Preface</a>
</li>
<li>
<a href="/sre/sre-book/chapters/part1" class="menu-buttons">Part I - Introduction</a>
</li>
<li>
<a href="/sre/sre-book/chapters/introduction" class="menu-buttons">1. Introduction</a>
</li>
<li>
<a href="/sre/sre-book/chapters/production-environment" class="menu-buttons">2. The Production Environment at Google, from the Viewpoint of an SRE</a>
</li>
<li>
<a href="/sre/sre-book/chapters/part2" class="menu-buttons">Part II - Principles</a>
</li>
<li>
<a href="/sre/sre-book/chapters/embracing-risk" class="menu-buttons">3. Embracing Risk</a>
</li>
<li>
<a href="/sre/sre-book/chapters/service-level-objectives" class="menu-buttons">4. Service Level Objectives</a>
</li>
<li>
<a href="/sre/sre-book/chapters/eliminating-toil" class="menu-buttons">5. Eliminating Toil</a>
</li>
<li class="active">
<a href="/sre/sre-book/chapters/monitoring-distributed-systems" class="menu-buttons">6. Monitoring Distributed Systems</a>
</li>
<li>
<a href="/sre/sre-book/chapters/automation-at-google" class="menu-buttons">7. The Evolution of Automation at Google</a>
</li>
<li>
<a href="/sre/sre-book/chapters/release-engineering" class="menu-buttons">8. Release Engineering</a>
</li>
<li>
<a href="/sre/sre-book/chapters/simplicity" class="menu-buttons">9. Simplicity</a>
</li>
<li>
<a href="/sre/sre-book/chapters/part3" class="menu-buttons">Part III - Practices</a>
</li>
<li>
<a href="/sre/sre-book/chapters/practical-alerting" class="menu-buttons">10. Practical Alerting</a>
</li>
<li>
<a href="/sre/sre-book/chapters/being-on-call" class="menu-buttons">11. Being On-Call</a>
</li>
<li>
<a href="/sre/sre-book/chapters/effective-troubleshooting" class="menu-buttons">12. Effective Troubleshooting</a>
</li>
<li>
<a href="/sre/sre-book/chapters/emergency-response" class="menu-buttons">13. Emergency Response</a>
</li>
<li>
<a href="/sre/sre-book/chapters/managing-incidents" class="menu-buttons">14. Managing Incidents</a>
</li>
<li>
<a href="/sre/sre-book/chapters/postmortem-culture" class="menu-buttons">15. Postmortem Culture: Learning from Failure</a>
</li>
<li>
<a href="/sre/sre-book/chapters/tracking-outages" class="menu-buttons">16. Tracking Outages</a>
</li>
<li>
<a href="/sre/sre-book/chapters/testing-reliability" class="menu-buttons">17. Testing for Reliability</a>
</li>
<li>
<a href="/sre/sre-book/chapters/software-engineering-in-sre" class="menu-buttons">18. Software Engineering in SRE</a>
</li>
<li>
<a href="/sre/sre-book/chapters/load-balancing-frontend" class="menu-buttons">19. Load Balancing at the Frontend</a>
</li>
<li>
<a href="/sre/sre-book/chapters/load-balancing-datacenter" class="menu-buttons">20. Load Balancing in the Datacenter</a>
</li>
<li>
<a href="/sre/sre-book/chapters/handling-overload" class="menu-buttons">21. Handling Overload</a>
</li>
<li>
<a href="/sre/sre-book/chapters/addressing-cascading-failures" class="menu-buttons">22. Addressing Cascading Failures</a>
</li>
<li>
<a href="/sre/sre-book/chapters/managing-critical-state" class="menu-buttons">23. Managing Critical State: Distributed Consensus for Reliability</a>
</li>
<li>
<a href="/sre/sre-book/chapters/distributed-periodic-scheduling" class="menu-buttons">24. Distributed Periodic Scheduling with Cron</a>
</li>
<li>
<a href="/sre/sre-book/chapters/data-processing-pipelines" class="menu-buttons">25. Data Processing Pipelines</a>
</li>
<li>
<a href="/sre/sre-book/chapters/data-integrity" class="menu-buttons">26. Data Integrity: What You Read Is What You Wrote</a>
</li>
<li>
<a href="/sre/sre-book/chapters/reliable-product-launches" class="menu-buttons">27. Reliable Product Launches at Scale</a>
</li>
<li>
<a href="/sre/sre-book/chapters/part4" class="menu-buttons">Part IV - Management</a>
</li>
<li>
<a href="/sre/sre-book/chapters/accelerating-sre-on-call" class="menu-buttons">28. Accelerating SREs to On-Call and Beyond</a>
</li>
<li>
<a href="/sre/sre-book/chapters/dealing-with-interrupts" class="menu-buttons">29. Dealing with Interrupts</a>
</li>
<li>
<a href="/sre/sre-book/chapters/operational-overload" class="menu-buttons">30. Embedding an SRE to Recover from Operational Overload</a>
</li>
<li>
<a href="/sre/sre-book/chapters/communication-and-collaboration" class="menu-buttons">31. Communication and Collaboration in SRE</a>
</li>
<li>
<a href="/sre/sre-book/chapters/evolving-sre-engagement-model" class="menu-buttons">32. The Evolving SRE Engagement Model</a>
</li>
<li>
<a href="/sre/sre-book/chapters/part5" class="menu-buttons">Part V - Conclusions</a>
</li>
<li>
<a href="/sre/sre-book/chapters/lessons-learned" class="menu-buttons">33. Lessons Learned from Other Industries</a>
</li>
<li>
<a href="/sre/sre-book/chapters/conclusion" class="menu-buttons">34. Conclusion</a>
</li>
<li>
<a href="/sre/sre-book/chapters/availability-table" class="menu-buttons">Appendix A. Availability Table</a>
</li>
<li>
<a href="/sre/sre-book/chapters/service-best-practices" class="menu-buttons">Appendix B. A Collection of Best Practices for Production Services</a>
</li>
<li>
<a href="/sre/sre-book/chapters/incident-document" class="menu-buttons">Appendix C. Example Incident State Document</a>
</li>
<li>
<a href="/sre/sre-book/chapters/postmortem" class="menu-buttons">Appendix D. Example Postmortem</a>
</li>
<li>
<a href="/sre/sre-book/chapters/launch-checklist" class="menu-buttons">Appendix E. Launch Coordination Checklist</a>
</li>
<li>
<a href="/sre/sre-book/chapters/production-meeting" class="menu-buttons">Appendix F. Example Production Meeting Minutes</a>
</li>
<li>
<a href="/sre/sre-book/chapters/bibliography" class="menu-buttons">Bibliography</a>
</li>
</ol>
</div>
</div>
<div id="maia-main" role="main">
<div class="maia-teleport" id="content"></div>
<div class="content">
<section data-type="chapter" id="chapter_monitoring">
<h1 class="heading jumptargets">
Monitoring Distributed Systems
</h1>
<p class="byline author">
Written by Rob Ewaschuk<br />
Edited by Betsy Beyer
</p>
<p>
Google’s SRE teams have some basic principles and best practices for building successful monitoring and alerting systems. This chapter offers guidelines for what issues should interrupt a human via a page, and how to deal with issues that aren’t serious enough to trigger a page.
</p>
<section data-type="sect1" id="definitions-2ksZhN">
<h1 class="heading jumptargets">
Definitions
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="terminology" id="id-DnC1SWFMhD"></a>There’s no uniformly shared vocabulary for discussing all topics related to monitoring. Even within Google, usage of the following terms varies, but the most common interpretations are listed here.
</p>
<dl>
<dt class="subheaders jumptargets" id="monitoring">
Monitoring
</dt>
<dd>
<p>
Collecting, processing, aggregating, and displaying real-time quantitative data about a system, such as query counts and types, error counts and types, processing times, and server lifetimes.
</p>
</dd>
<dt class="subheaders jumptargets" id="white-box-monitoring">
White-box monitoring
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="white-box monitoring" id="id-9nCjSDS4tZILhX"></a>Monitoring based on metrics exposed by the internals of the system, including logs, interfaces like the Java Virtual Machine Profiling Interface, or an HTTP handler that emits internal statistics.
</p>
</dd>
<dt class="subheaders jumptargets" id="black-box-monitoring">
Black-box monitoring
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCxSrSgTWIdhb"></a>Testing externally visible behavior as a user would see it.
</p>
</dd>
<dt class="subheaders jumptargets" id="dashboard">
Dashboard
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="defined" id="id-VMCPS2SribIkh4"></a>An application (usually web-based) that provides a summary view of a service’s core metrics. A dashboard may have filters, selectors, and so on, but is prebuilt to expose the metrics most important to its users. The dashboard might also display team information such as ticket queue length, a list of high-priority bugs, the current on-call engineer for a given area of responsibility, or recent pushes.
</p>
</dd>
<dt class="subheaders jumptargets" id="alert">
Alert
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="alerts" data-secondary="defined" id="id-wqC7SvSPUAIVhQ"></a>A notification intended to be read by a human and that is pushed to a system such as a bug or ticket queue, an email alias, or a pager. Respectively, these alerts are classified as <em>tickets</em>, <em>email alerts</em>,<sup><a class="jumptarget" data-type="noteref" id="id-LvQuvtYS7UvI8h4-marker" href="#id-LvQuvtYS7UvI8h4">22</a></sup> and <em>pages</em>.
</p>
</dd>
<dt class="subheaders jumptargets" id="root-cause">
Root cause
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="root cause" data-secondary="defined" id="id-PnCpSaSKsgIjho"></a>A defect in a software or human system that, if repaired, instills confidence that this event won’t happen again in the same way. A given incident might have multiple root causes: for example, perhaps it was caused by a combination of insufficient process automation, software that crashed on bogus input, <em>and</em> insufficient testing of the script used to generate the configuration. Each of these factors might stand alone as a root cause, and each should be repaired.
</p>
</dd>
<dt class="subheaders jumptargets" id="node-and-machine">
Node and machine
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="machines" data-secondary="defined" id="id-XmC9SkSlfnI1hK"></a>Used interchangeably to indicate a single instance of a running kernel in either a physical server, virtual machine, or container. There might be multiple <em>services</em> worth monitoring on a single machine. The services may either be:
</p>
<ul>
<li>Related to each other: for example, a caching server and a web server
</li>
<li>Unrelated services sharing hardware: for example, a code repository and a master for a configuration system like <a href="https://puppetlabs.com/puppet/puppet-open-source" target="_blank">Puppet</a> or <a href="https://www.chef.io/chef/" target="_blank">Chef</a>
</li>
</ul>
</dd>
<dt class="subheaders jumptargets" id="push">
Push
</dt>
<dd>
<p>
Any change to a service’s running software or its configuration.
</p>
</dd>
</dl>
</section>
<section data-type="sect1" id="why-monitor-pWsBTZ">
<h1 class="heading jumptargets">
Why Monitor?
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="benefits of monitoring" id="id-kVCkSpFnTl"></a>There are many reasons to monitor a system, including:
</p>
<dl>
<dt class="subheaders jumptargets" id="analyzing-long-term-trends">
Analyzing long-term trends
</dt>
<dd>
<p>
How big is my database and how fast is it growing? How quickly is my daily-active user count growing?
</p>
</dd>
<dt class="subheaders jumptargets" id="comparing-over-time-or-experiment-groups">
Comparing over time or experiment groups
</dt>
<dd>
<p>
Are queries faster with Acme Bucket of Bytes 2.72 versus Ajax DB 3.14? How much better is my memcache hit rate with an extra node? Is my site slower than it was last week?
</p>
</dd>
<dt class="subheaders jumptargets" id="alerting">
Alerting
</dt>
<dd>
<p>
Something is broken, and somebody needs to fix it right now! Or, something might break soon, so somebody should look soon.
</p>
</dd>
<dt class="subheaders jumptargets" id="building-dashboards">
Building dashboards
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="dashboards" data-secondary="benefits of" id="id-rjCXSOS0iDIGT8"></a>Dashboards should answer basic questions about your service, and normally include some form of the four golden signals (discussed in <a data-type="xref" href="#xref_monitoring_golden-signals">The Four Golden Signals</a>).
</p>
</dd>
<dt class="subheaders jumptargets" id="conducting-ad-hoc-retrospective-analysis-ie-debugging">
Conducting <i class="italic">ad hoc</i> retrospective analysis (i.e., debugging)
</dt>
<dd>
<p>
Our latency just shot up; what else happened around the same time?
</p>
</dd>
</dl>
<p>
System monitoring is also helpful in supplying raw input into business analytics and in facilitating analysis of security breaches. Because this book focuses on the engineering domains in which SRE has particular expertise, we won’t discuss these applications of monitoring here.
</p>
<p>
Monitoring and alerting enables a system to tell us when it’s broken, or perhaps to tell us what’s about to break. When the system isn’t able to automatically fix itself, we want a human to investigate the alert, determine if there’s a real problem at hand, mitigate the problem, and determine the root cause of the problem. Unless you’re performing security auditing on very narrowly scoped components of a system, you should never trigger an alert simply because "something seems a bit weird."
</p>
<p>
Paging a human is a quite expensive use of an employee’s time. If an employee is at work, a page interrupts their workflow. If the employee is at home, a page interrupts their personal time, and perhaps even their sleep. When pages occur too frequently, employees second-guess, skim, or even ignore incoming alerts, sometimes even ignoring a "real" page that’s masked by the noise. Outages can be prolonged because other noise interferes with a rapid diagnosis and fix. Effective alerting systems have good signal and very low noise.
</p>
</section>
<section data-type="sect1" id="setting-reasonable-expectations-for-monitoring-o8svcM">
<h1 class="heading jumptargets">
Setting Reasonable Expectations for Monitoring
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="setting expectations for" id="id-4nCqSYFQcE"></a>Monitoring a complex application is a significant engineering endeavor in and of itself. Even with substantial existing infrastructure for instrumentation, collection, display, and alerting in place, a Google SRE team with 10–12 members typically has one or sometimes two members whose primary assignment is to build and maintain monitoring systems for their service. This number has decreased over time as we generalize and centralize common monitoring infrastructure, but every SRE team typically has at least one “monitoring person.” (That being said, while it can be fun to have access to traffic graph dashboards and the like, SRE teams carefully avoid any situation that requires someone to “stare at a screen to watch for problems.”)
</p>
<p>
<a data-type="indexterm" data-primary="post hoc analysis" id="id-JnCDSjIVcG"></a>In general, Google has trended toward simpler and faster monitoring systems, with better tools for <em>post hoc</em> analysis. We avoid "magic" systems that try to learn thresholds or automatically detect causality. Rules that detect unexpected changes in end-user request rates are one counterexample; while these rules are still kept as simple as possible, they give a very quick detection of a very simple, specific, severe anomaly. Other uses of monitoring data such as capacity planning and traffic prediction can tolerate more fragility, and thus, more complexity. Observational experiments conducted over a very long time horizon (months or years) with a low sampling rate (hours or days) can also often tolerate more fragility because occasional missed samples won’t hide a long-running trend.
</p>
<p>
<a data-type="indexterm" data-primary="dependency hierarchies" id="id-9nCjSOtmcj"></a>Google SRE has experienced only limited success with complex dependency hierarchies. We seldom use rules such as, "If I know the database is slow, alert for a slow database; otherwise, alert for the website being generally slow." Dependency-reliant rules usually pertain to very stable parts of our system, such as our system for draining user traffic away from a datacenter. For example, "If a datacenter is drained, then don’t alert me on its latency" is one common datacenter alerting rule. Few teams at Google maintain complex dependency hierarchies because our infrastructure has a steady rate of continuous refactoring.
</p>
<p>
Some of the ideas described in this chapter are still aspirational: there is always room to move more rapidly from symptom to root cause(s), especially in ever-changing systems. So while this chapter sets out some goals for monitoring systems, and some ways to achieve these goals, it’s important that monitoring systems—especially the critical path from the onset of a production problem, through a page to a human, through basic triage and deep debugging—be kept simple and comprehensible by everyone on the team.
</p>
<p>
Similarly, to keep noise low and signal high, the elements of your monitoring system that direct to a pager need to be very simple and robust. Rules that generate alerts for humans should be simple to understand and represent a clear failure.
</p>
</section>
<section data-type="sect1" id="symptoms-versus-causes-g0sEi4">
<h1 class="heading jumptargets">
Symptoms Versus Causes
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="symptoms vs. causes" id="id-JnCDSlFmiG"></a>Your monitoring system should address two questions: what’s broken, and why?
</p>
<p>
The "what’s broken" indicates the symptom; the "why" indicates a (possibly intermediate) cause. <a data-type="xref" href="#table_monitoring_symptoms">Table 6-1</a> lists some hypothetical symptoms and corresponding causes.
</p>
<table id="table_monitoring_symptoms" class="pagebreak-before">
<caption class="jumptarget">
<span class="label">Table 6-1.</span> Example symptoms and causes
</caption>
<thead>
<tr>
<th>
<strong>Symptom</strong>
</th>
<th>
<strong>Cause</strong>
</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<p>
<strong>I’m serving HTTP 500s or 404s</strong>
</p>
</td>
<td>
<p>
Database servers are refusing connections
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>My responses are slow</strong>
</p>
</td>
<td>
<p>
CPUs are overloaded by a bogosort, or an Ethernet cable is crimped under a rack, visible as partial packet loss
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Users in Antarctica aren’t receiving animated cat GIFs</strong>
</p>
</td>
<td>
<p>
Your Content Distribution Network hates scientists and felines, and thus blacklisted some client IPs
</p>
</td>
</tr>
<tr>
<td>
<p>
<strong>Private content is world-readable</strong>
</p>
</td>
<td>
<p>
A new software push caused ACLs to be forgotten and allowed all requests
</p>
</td>
</tr>
</tbody>
</table>
<p>
"What" versus "why" is one of the most important distinctions in writing good monitoring with maximum signal and minimum noise.
</p>
</section>
<section data-type="sect1" id="black-box-versus-white-box-q8sJuw">
<h1 class="heading jumptargets">
Black-Box Versus White-Box
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="blackbox vs. whitebox" id="id-9nCjSvFVuj"></a><a data-type="indexterm" data-primary="white-box monitoring" id="id-ZbC1FMFEu7"></a><a data-type="indexterm" data-primary="black-box monitoring" id="id-zdCXIGFvuy"></a>We combine heavy use of white-box monitoring with modest but critical uses of black-box monitoring. The simplest way to think about black-box monitoring versus white-box monitoring is that black-box monitoring is symptom-oriented and represents active—not predicted—problems: "The system isn’t working correctly, right now." White-box monitoring depends on the ability to inspect the innards of the system, such as logs or HTTP endpoints, with instrumentation. White-box monitoring therefore allows detection of imminent problems, failures masked by retries, and so forth.
</p>
<p>
Note that in a multilayered system, one person’s symptom is another person’s cause. For example, suppose that a database’s performance is slow. Slow database reads are a symptom for the database SRE who detects them. However, for the frontend SRE observing a slow website, the same slow database reads are a cause. Therefore, white-box monitoring is sometimes symptom-oriented, and sometimes cause-oriented, depending on just how informative your white-box is.
</p>
<p>
When collecting telemetry for debugging, white-box monitoring is essential. If web servers seem slow on database-heavy requests, you need to know both how fast the web server perceives the database to be, and how fast the database believes itself to be. Otherwise, you can’t distinguish an actually slow database server from a network problem between your web server and your database.
</p>
<p>
For paging, black-box monitoring has the key benefit of forcing discipline to only nag a human when a problem is both already ongoing and contributing to real symptoms. On the other hand, for not-yet-occurring but imminent problems, black-box monitoring is fairly useless.
</p>
</section>
<section data-type="sect1" id="xref_monitoring_golden-signals">
<h1 class="heading jumptargets">
The Four Golden Signals
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="four golden signals of" id="id-ZbCxSMFjU7"></a>The four golden signals of monitoring are latency, traffic, errors, and saturation. If you can only measure four metrics of your user-facing system, focus on these four.
</p>
<dl>
<dt class="subheaders jumptargets" id="latency">
Latency
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="service latency" data-secondary="monitoring for" id="id-yYCASJS9FKIWUb"></a><a data-type="indexterm" data-primary="latency" data-secondary="monitoring for" id="id-VMCpF2SXFbIwU4"></a><a data-type="indexterm" data-primary="request latency" id="id-rjCeIOSKFDIaU8"></a><a data-type="indexterm" data-primary="user requests" data-secondary="request latency monitoring" id="id-wqCDtvSGFAIMUQ"></a>The time it takes to service a request. It’s important to distinguish between the latency of successful requests and the latency of failed requests. For example, an HTTP 500 error triggered due to loss of connection to a database or other critical backend might be served very quickly; however, as an HTTP 500 error indicates a failed request, factoring 500s into your overall latency might result in misleading calculations. On the other hand, a slow error is even worse than a fast error! Therefore, it’s important to track error latency, as opposed to just filtering out errors.
</p>
</dd>
<dt class="subheaders jumptargets" id="traffic">
Traffic
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="user requests" data-secondary="traffic analysis" id="id-rjCXSOSxtDIaU8"></a><a data-type="indexterm" data-primary="traffic analysis" id="id-wqC4FvSBtAIMUQ"></a>A measure of how much demand is being placed on your system, measured in a high-level system-specific metric. For a web service, this measurement is usually HTTP requests per second, perhaps broken out by the nature of the requests (e.g., static versus dynamic content). For an audio streaming system, this measurement might focus on network I/O rate or concurrent sessions. For a key-value storage system, this measurement might be transactions and retrievals per <span class="keep-together">second</span>.
</p>
</dd>
<dt class="subheaders jumptargets" id="errors">
Errors
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="error rates" id="id-x1C4SjSlTLIMUJ"></a><a data-type="indexterm" data-primary="user requests" data-secondary="monitoring failures" id="id-PnCxFaS0TgIVUo"></a>The rate of requests that fail, either explicitly (e.g., HTTP 500s), implicitly (for example, an HTTP 200 success response, but coupled with the wrong content), or by policy (for example, "If you committed to one-second response times, any request over one second is an error"). Where protocol response codes are insufficient to express all failure conditions, secondary (internal) protocols may be necessary to track partial failure modes. Monitoring these cases can be drastically different: catching HTTP 500s at your load balancer can do a decent job of catching all completely failed requests, while only end-to-end system tests can detect that you’re serving the wrong content.
</p>
</dd>
<dt class="subheaders jumptargets" id="saturation">
Saturation
</dt>
<dd>
<p>
<a data-type="indexterm" data-primary="saturation" id="id-OnCNS2S4iDIYU8"></a>How "full" your service is. A measure of your system fraction, emphasizing the resources that are most constrained (e.g., in a memory-constrained system, show memory; in an I/O-constrained system, show I/O). Note that many systems degrade in performance before they achieve 100% utilization, so having a utilization target is essential.
</p>
<p>
In complex systems, saturation can be supplemented with higher-level load measurement: can your service properly handle double the traffic, handle only 10% more traffic, or handle even less traffic than it currently receives? For very simple services that have no parameters that alter the complexity of the request (e.g., "Give me a nonce" or "I need a globally unique monotonic integer") that rarely change configuration, a static value from a load test might be adequate. As discussed in the previous paragraph, however, most services need to use indirect signals like CPU utilization or network bandwidth that have a known upper bound. Latency increases are often a leading indicator of saturation. Measuring your 99th percentile response time over some small window (e.g., one minute) can give a very early signal of saturation.
</p>
<p>
Finally, saturation is also concerned with predictions of impending saturation, such as "It looks like your database will fill its hard drive in 4 hours."
</p>
</dd>
</dl>
<p>
If you measure all four golden signals and page a human when one signal is problematic (or, in the case of saturation, nearly problematic), your service will be at least decently covered by monitoring.
</p>
</section>
<section data-type="sect1" id="worrying-about-your-tail-or-instrumentation-and-performance-Yms9Ck">
<h1 class="heading jumptargets">
Worrying About Your Tail (or, Instrumentation and Performance)
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="instrumentation and performance" id="id-zdCxSGFQCy"></a><a data-type="indexterm" data-primary="performance" data-secondary="monitoring" id="id-yYCyFpFdCr"></a>When building a monitoring system from scratch, it’s tempting to design a system based upon the mean of some quantity: the mean latency, the mean CPU usage of your nodes, or the mean fullness of your databases. The danger presented by the latter two cases is obvious: CPUs and databases can easily be utilized in a very imbalanced way. The same holds for latency. If you run a web service with an average latency of 100 ms at 1,000 requests per second, 1% of requests might easily take 5 seconds.<sup><a class="jumptarget" data-type="noteref" id="id-QQLuAIXFxCz-marker" href="#id-QQLuAIXFxCz">23</a></sup> If your users depend on several such web services to render their page, the 99th percentile of one backend can easily become the median response of your <span class="keep-together">frontend</span>.
</p>
<p>
The simplest way to differentiate between a slow average and a very slow "tail" of requests is to collect request counts bucketed by latencies (suitable for rendering a histogram), rather than actual latencies: how many requests did I serve that took between 0 ms and 10 ms, between 10 ms and 30 ms, between 30 ms and 100 ms, between 100 ms and 300 ms, and so on? Distributing the histogram boundaries approximately exponentially (in this case by factors of roughly 3) is often an easy way to visualize the distribution of your requests.
</p>
</section>
<section data-type="sect1" id="choosing-an-appropriate-resolution-for-measurements-vJsBsE">
<h1 class="heading jumptargets">
Choosing an Appropriate Resolution for Measurements
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="resolution" id="id-yYCASpFxsr"></a>Different aspects of a system should be measured with different levels of granularity. For example:
</p>
<ul>
<li>Observing CPU load over the time span of a minute won’t reveal even quite long-lived spikes that drive high tail latencies.
</li>
<li>On the other hand, for a web service targeting no more than 9 hours aggregate downtime per year (99.9% annual uptime), probing for a 200 (success) status more than once or twice a minute is probably unnecessarily frequent.
</li>
<li>Similarly, checking hard drive fullness for a service targeting 99.9% availability more than once every 1–2 minutes is probably unnecessary.
</li>
</ul>
<p>
Take care in how you structure the granularity of your measurements. Collecting per-second measurements of CPU load might yield interesting data, but such frequent measurements may be very expensive to collect, store, and analyze. If your monitoring goal calls for high resolution but doesn’t require extremely low latency, you can reduce these costs by performing internal sampling on the server, then configuring an external system to collect and aggregate that distribution over time or across servers. You might:
</p>
<ol>
<li>Record the current CPU utilization each second.
</li>
<li>Using buckets of 5% granularity, increment the appropriate CPU utilization bucket each second.
</li>
<li>Aggregate those values every minute.
</li>
</ol>
<p>
This strategy allows you to observe brief CPU hotspots without incurring very high cost due to collection and retention.
</p>
</section>
<section data-type="sect1" id="as-simple-as-possible-no-simpler-lqskHx">
<h1 class="heading jumptargets">
As Simple as Possible, No Simpler
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="avoiding complexity in" id="id-VMCPSrFpHm"></a>Piling all these requirements on top of each other can add up to a very complex monitoring system—your system might end up with the following levels of complexity:
</p>
<ul>
<li>Alerts on different latency thresholds, at different percentiles, on all kinds of different metrics
</li>
<li>Extra code to detect and expose possible causes
</li>
<li>Associated dashboards for each of these possible causes
</li>
</ul>
<p>
The sources of potential complexity are never-ending. Like all software systems, monitoring can become so complex that it’s fragile, complicated to change, and a maintenance burden.
</p>
<p>
Therefore, design your monitoring system with an eye toward simplicity. In choosing what to monitor, keep the following guidelines in mind:
</p>
<ul>
<li>The rules that catch real incidents most often should be as simple, predictable, and reliable as possible.
</li>
<li>Data collection, aggregation, and alerting configuration that is rarely exercised (e.g., less than once a quarter for some SRE teams) should be up for removal.
</li>
<li>Signals that are collected, but not exposed in any prebaked dashboard nor used by any alert, are candidates for removal.
</li>
</ul>
<p>
In Google’s experience, basic collection and aggregation of metrics, paired with alerting and dashboards, has worked well as a relatively standalone system. (In fact Google’s monitoring system is broken up into several binaries, but typically people learn about all aspects of these binaries.) It can be tempting to combine monitoring with other aspects of inspecting complex systems, such as detailed system profiling, single-process debugging, tracking details about exceptions or crashes, load testing, log collection and analysis, or traffic inspection. While most of these subjects share commonalities with basic monitoring, blending together too many results in overly complex and fragile systems. As in many other aspects of software engineering, maintaining distinct systems with clear, simple, loosely coupled points of integration is a better strategy (for example, using web APIs for pulling summary data in a format that can remain constant over an extended period of time).
</p>
</section>
<section data-type="sect1" id="tying-these-principles-together-nqsJfw">
<h1 class="heading jumptargets">
Tying These Principles Together
</h1>
<p>
The principles discussed in this chapter can be tied together into a philosophy on monitoring and alerting that’s widely endorsed and followed within Google SRE teams. While this monitoring philosophy is a bit aspirational, it’s a good starting point for writing or reviewing a new alert, and it can help your organization ask the right questions, regardless of the size of your organization or the complexity of your service or system.
</p>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="creating rules for" id="id-wqC7SDIvfj"></a>When creating rules for monitoring and alerting, asking the following questions can help you avoid false positives and pager burnout:<sup><a class="jumptarget" data-type="noteref" id="id-a82udF8IBfx-marker" href="#id-a82udF8IBfx">24</a></sup>
</p>
<ul>
<li>Does this rule detect <em>an otherwise undetected condition</em> that is urgent, actionable, and actively or imminently user-visible?<sup><a class="jumptarget" data-type="noteref" id="id-0vYuEFpSjSMtLfG-marker" href="#id-0vYuEFpSjSMtLfG">25</a></sup>
</li>
<li>Will I ever be able to ignore this alert, knowing it’s benign? When and why will I be able to ignore this alert, and how can I avoid this scenario?
</li>
<li>Does this alert definitely indicate that users are being negatively affected? Are there detectable cases in which users aren’t being negatively impacted, such as drained traffic or test deployments, that should be filtered out?
</li>
<li>Can I take action in response to this alert? Is that action urgent, or could it wait until morning? Could the action be safely automated? Will that action be a long-term fix, or just a short-term workaround?
</li>
<li>Are other people getting paged for this issue, therefore rendering at least one of the pages unnecessary?
</li>
</ul>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="monitoring philosophy" id="id-PnCpSwhJfa"></a>These questions reflect a fundamental philosophy on pages and pagers:
</p>
<ul>
<li>Every time the pager goes off, I should be able to react with a sense of urgency. I can only react with a sense of urgency a few times a day before I become fatigued.
</li>
<li>Every page should be actionable.
</li>
<li>Every page response should require intelligence. If a page merely merits a robotic response, it shouldn’t be a page.
</li>
<li>Pages should be about a novel problem or an event that hasn’t been seen before.
</li>
</ul>
<p>
Such a perspective dissipates certain distinctions: if a page satisfies the preceding four bullets, it’s irrelevant whether the page is triggered by white-box or black-box monitoring. This perspective also amplifies certain distinctions: it’s better to spend much more effort on catching symptoms than causes; when it comes to causes, only worry about very definite, very imminent causes.
</p>
</section>
<section data-type="sect1" id="monitoring-for-the-long-term-NbsNS8">
<h1 class="heading jumptargets">
Monitoring for the Long Term
</h1>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="challenges of" id="id-wqC7SPFMSj"></a>In modern production systems, monitoring systems track an ever-evolving system with changing software architecture, load characteristics, and performance targets. An alert that’s currently exceptionally rare and hard to automate might become frequent, perhaps even meriting a hacked-together script to resolve it. At this point, someone should find and eliminate the root causes of the problem; if such resolution isn’t possible, the alert response deserves to be fully automated.
</p>
<p>
It’s important that decisions about monitoring be made with long-term goals in mind. Every page that happens today distracts a human from improving the system for tomorrow, so there is often a case for taking a short-term hit to availability or performance in order to improve the long-term outlook for the system. Let’s take a look at two case studies that illustrate this trade-off.
</p>
<section data-type="sect2" id="bigtable-sre-a-tale-of-over-alerting-dbsXtjSM">
<h2 class="subheaders jumptargets">
Bigtable SRE: A Tale of Over-Alerting
</h2>
<p>
<a data-type="indexterm" id="MDSbig6" data-primary="monitoring distributed systems" data-secondary="case studies"></a><a data-type="indexterm" data-primary="Bigtable" id="id-XmCpFOFytySv"></a>Google’s internal infrastructure is typically offered and measured against a service level objective (SLO; see <a data-type="xref" href="/sre/sre-book/chapters/service-level-objectives">Service Level Objectives</a>). Many years ago, the Bigtable service’s SLO was based on a synthetic well-behaved client’s mean performance. Because of problems in Bigtable and lower layers of the storage stack, the mean performance was driven by a "large" tail: the worst 5% of requests were often significantly slower than the rest.
</p>
<p>
Email alerts were triggered as the SLO approached, and paging alerts were triggered when the SLO was exceeded. Both types of alerts were firing voluminously, consuming unacceptable amounts of engineering time: the team spent significant amounts of time triaging the alerts to find the few that were really actionable, and we often missed the problems that actually affected users, because so few of them did. Many of the pages were non-urgent, due to well-understood problems in the infrastructure, and had either rote responses or received no response.
</p>
<p>
To remedy the situation, the team used a three-pronged approach: while making great efforts to improve the performance of Bigtable, we also temporarily dialed back our SLO target, using the 75th percentile request latency. We also disabled email alerts, as there were so many that spending time diagnosing them was infeasible.
</p>
<p>
This strategy gave us enough breathing room to actually fix the longer-term problems in Bigtable and the lower layers of the storage stack, rather than constantly fixing tactical problems. On-call engineers could actually accomplish work when they weren’t being kept up by pages at all hours. Ultimately, temporarily backing off on our alerts allowed us to make faster progress toward a better service.
</p>
</section>
<section data-type="sect2" id="gmail-predictable-scriptable-responses-from-humans-BVs1h4SD">
<h2 class="subheaders jumptargets">
Gmail: Predictable, Scriptable Responses from Humans
</h2>
<p>
<a data-type="indexterm" data-primary="Gmail" id="id-XmC9SOFZhySv"></a>In the very early days of Gmail, the service was built on a retrofitted distributed process management system called Workqueue, which was originally created for batch processing of pieces of the search index. Workqueue was "adapted" to long-lived processes and subsequently applied to Gmail, but certain bugs in the relatively opaque codebase in the scheduler proved hard to beat.
</p>
<p>
At that time, the Gmail monitoring was structured such that alerts fired when individual tasks were “de-scheduled” by Workqueue. This setup was less than ideal because even at that time, Gmail had many, many thousands of tasks, each task representing a fraction of a percent of our users. We cared deeply about providing a good user experience for Gmail users, but such an alerting setup was unmaintainable.
</p>
<p>
To address this problem, Gmail SRE built a tool that helped “poke” the scheduler in just the right way to minimize impact to users. The team had several discussions about whether or not we should simply automate the entire loop from detecting the problem to nudging the rescheduler, until a better long-term solution was achieved, but some worried this kind of workaround would delay a real fix.
</p>
<p>
This kind of tension is common within a team, and often reflects an underlying mistrust of the team’s self-discipline: while some team members want to implement a “hack” to allow time for a proper fix, others worry that a hack will be forgotten or that the proper fix will be deprioritized indefinitely. This concern is credible, as it’s easy to build layers of unmaintainable technical debt by patching over problems instead of making real fixes. Managers and technical leaders play a key role in implementing true, long-term fixes by supporting and prioritizing potentially time-consuming long-term fixes even when the initial “pain” of paging subsides.
</p>
<p>
Pages with rote, algorithmic responses should be a red flag. Unwillingness on the part of your team to automate such pages implies that the team lacks confidence that they can clean up their technical debt. This is a major problem worth escalating.<a data-type="indexterm" data-primary="" data-startref="MDSbig6" id="id-oPCASqT2hLSk"></a>
</p>
</section>
<section data-type="sect2" id="the-long-run-MQsWTMS7">
<h2 class="subheaders jumptargets">
The Long Run
</h2>
<p>
<a data-type="indexterm" data-primary="monitoring distributed systems" data-secondary="short- vs. long-term availability" id="id-jyCxSoFETNSd"></a>A common theme connects the previous examples of Bigtable and Gmail: a tension between short-term and long-term availability. Often, sheer force of effort can help a rickety system achieve high availability, but this path is usually short-lived and fraught with burnout and dependence on a small number of heroic team members. Taking a controlled, short-term decrease in availability is often a painful, but strategic trade for the long-run stability of the system. It’s important not to think of every page as an event in isolation, but to consider whether the overall <em>level</em> of paging leads toward a healthy, appropriately available system with a healthy, viable team and long-term outlook. We review statistics about page frequency (usually expressed as incidents per shift, where an incident might be composed of a few related pages) in quarterly reports with management, ensuring that decision makers are kept up to date on the pager load and overall health of their teams.
</p>
</section>
</section>
<section data-type="sect1" id="conclusion-8ksvFj">
<h1 class="heading jumptargets">
Conclusion
</h1>
<p>
A healthy monitoring and alerting pipeline is simple and easy to reason about. It focuses primarily on symptoms for paging, reserving cause-oriented heuristics to serve as aids to debugging problems. Monitoring symptoms is easier the further "up" your stack you monitor, though monitoring saturation and performance of subsystems such as databases often must be performed directly on the subsystem itself. Email alerts are of very limited value and tend to easily become overrun with noise; instead, you should favor a dashboard that monitors all ongoing subcritical problems for the sort of information that typically ends up in email alerts. A dashboard might also be paired with a log, in order to analyze historical correlations.
</p>
<p>
Over the long haul, achieving a successful on-call rotation and product includes choosing to alert on symptoms or imminent real problems, adapting your targets to goals that are actually achievable, and making sure that your monitoring supports rapid diagnosis.
</p>
</section>
<div class="footnotes" data-type="footnotes">
<p data-type="footnote" id="id-LvQuvtYS7UvI8h4">
<sup><a class="jumptargets" href="#id-LvQuvtYS7UvI8h4-marker">22</a></sup>Sometimes known as "alert spam," as they are rarely read or acted on.
</p>
<p data-type="footnote" id="id-QQLuAIXFxCz">
<sup><a class="jumptargets" href="#id-QQLuAIXFxCz-marker">23</a></sup>If 1% of your requests are 50x the average, it means that the rest of your requests are about twice as fast as the average. But if you’re not measuring your distribution, the idea that most of your requests are near the mean is just hopeful thinking.
</p>
<p data-type="footnote" id="id-a82udF8IBfx">
<sup><a class="jumptargets" href="#id-a82udF8IBfx-marker">24</a></sup>See <em>Applying Cardiac Alarm Management Techniques to Your On-Call</em> <a data-type="xref" href="/sre/sre-book/chapters/bibliography#Hol14" target="_blank">[Hol14]</a> for an example of alert fatigue in another context.
</p>
<p data-type="footnote" id="id-0vYuEFpSjSMtLfG">
<sup><a class="jumptargets" href="#id-0vYuEFpSjSMtLfG-marker">25</a></sup>Zero-redundancy (<em>N</em> + 0) situations count as imminent, as do "nearly full" parts of your service! For more details about the concept of redundancy, see <a href="https://en.wikipedia.org/wiki/N%2B1_redundancy" target="_blank"><em class="hyperlink">https://en.wikipedia.org/wiki/N%2B1_redundancy</em></a>.
</p>
</div>
</section>
</div>
</div>
<div class="footer">
<div class="maia-aux">
<div class="previous">
<a href="/sre/sre-book/chapters/eliminating-toil">
<p class="footer-caption">
Previous
</p>
<p class="chapter-link">
Chapter 5 - Eliminating Toil
</p></a>
</div>
<div class="next">
<a href="/sre/sre-book/chapters/automation-at-google">
<p class="footer-caption">
Next
</p>
<p class="chapter-link">
Chapter 7 - The Evolution of Automation at Google
</p></a>
</div>
<p class="footer-link">
Copyright © 2017 Google, Inc. Published by O'Reilly Media, Inc. Licensed under <a href="https://creativecommons.org/licenses/by-nc-nd/4.0/" target="_blank">CC BY-NC-ND 4.0</a>
</p>
</div>
</div>
</main>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular-animate.min.js"></script>
<script src="//ajax.googleapis.com/ajax/libs/angularjs/1.6.6/angular-touch.min.js"></script>
<script src="/sre/sre-book/static/js/index.min.js?cache=5b7f90b"></script>
</body>
</html>

16
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/guardian-1/expected-images.json vendored Normal file
View File

@ -0,0 +1,16 @@
[
"https:\/\/i.guim.co.uk\/img\/media\/df84b519a877d652e950ecd4248320eec985934e\/0_320_4800_2880\/master\/4800.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctZGVmYXVsdC5wbmc&s=af41545b21b557e4f57dd4221b6a7f89",
"https:\/\/i.guim.co.uk\/img\/media\/05cb692c634cd90e5411aab92ca3e649474ff786\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=575838a657b26493e956c7f84b058080",
"https:\/\/i.guim.co.uk\/img\/media\/98c683a7df9c83b2c13de2d93ca1825199ed5150\/0_0_4800_3166\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=2f198e1958f140f3ac664a3fdd87177c",
"https:\/\/i.guim.co.uk\/img\/media\/0447972cf47ca67882fcfc648edf7e574b0853bc\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=718132fd888108a18383c24a8425523b",
"https:\/\/i.guim.co.uk\/img\/media\/416800b8d06039780c3e6de28564e6f277b4e7b7\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=c5a53178ebbe54490c97fad6b5e032c4",
"https:\/\/i.guim.co.uk\/img\/media\/8c207197c0a9e6f407dcddfded5f868a142c9cab\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=0dd659ae339b1aea9a99ed7f6f8eadeb",
"https:\/\/i.guim.co.uk\/img\/media\/0d13adeb0790af5c5fa317ce477c323d0e1c773c\/0_0_4800_2334\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=41b88ee343b9be76688b88443c7a8958",
"https:\/\/i.guim.co.uk\/img\/media\/4973b41f53b8ade499f99a305b01157eca659ca5\/0_0_1200_900\/master\/1200.jpg?width=300&quality=85&auto=format&fit=max&s=7c255bf6f8a27c56365a86813cdd1517",
"https:\/\/i.guim.co.uk\/img\/media\/d1941b6a6908314fab28f44da222a4c892213341\/0_0_4800_3120\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=76a26a289728e3d625e57d32eced57d8",
"https:\/\/i.guim.co.uk\/img\/media\/14b462f5d9489def554e0f9f436f13aec332f7b8\/0_0_4278_4800\/master\/4278.jpg?width=300&quality=85&auto=format&fit=max&s=a45ec7578a392eec201d2f6920b609a0",
"https:\/\/i.guim.co.uk\/img\/media\/e2cf54c36f17c6894844ea0cdd4346288a002da9\/915_0_3172_3189\/master\/3172.jpg?width=300&quality=85&auto=format&fit=max&s=0bd8e9f51bdf79a6e0a15ed176cfb57d",
"https:\/\/i.guim.co.uk\/img\/media\/b5f3736b2ba2ef4df364258b0efcaba26f571d6e\/0_0_4800_3073\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=7fcadc35b3a44ebafe3c469c6e89241d",
"https:\/\/i.guim.co.uk\/img\/media\/d5aaf60e3a427f278747acf0c3e7ba39b39ef923\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=1b74b488aedb864287ff160f86d74c9d",
"https:\/\/i.guim.co.uk\/img\/media\/3766106f73e858d5b140ae3cdd2eef84060180cd\/0_0_4800_3200\/master\/4800.jpg?width=300&quality=85&auto=format&fit=max&s=d2f5bb7c3c3642ac8733ca40509f6e20"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/guardian-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Eleanor Ainge Roy",
"Direction": null,
"Excerpt": "New Zealand\u2019s whale whisperers worry that manmade changes in the ocean are behind the spike in beachings",
"Image": "https:\/\/i.guim.co.uk\/img\/media\/df84b519a877d652e950ecd4248320eec985934e\/0_320_4800_2880\/master\/4800.jpg?width=1200&height=630&quality=85&auto=format&fit=crop&overlay-align=bottom%2Cleft&overlay-width=100p&overlay-base64=L2ltZy9zdGF0aWMvb3ZlcmxheXMvdGctZGVmYXVsdC5wbmc&s=af41545b21b557e4f57dd4221b6a7f89",
"Title": "'What is the sea telling us?': M\u0101ori tribes fearful over whale strandings | Eleanor Ainge Roy",
"SiteName": "the Guardian"
}

297
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/guardian-1/expected.html vendored Normal file
View File

@ -0,0 +1,297 @@
<div itemprop="articleBody" data-test-id="article-review-body">
<p>
<span><span>W</span></span>hale whisperer Hori Parata was just seven years old when he attended his first mass stranding, a beaching of porpoises in New Zealand’s Northland, their cries screeching through the air on the deserted stretch of sand.
</p>
<p>
Seven decades later, Parata, 75, has now overseen more than 500 strandings and is renowned in <a href="https://www.theguardian.com/world/newzealand" data-link-name="auto-linked-tag" data-component="auto-linked-tag">New Zealand</a> as the leading Māori whale expert, called on by tribes around the country for cultural guidance as marine strandings become increasingly complex and fatal.
</p>
<p>
“Man’s greed in the ocean is hurting the whales,” says Parata, a fierce and uncompromising elder of the Ngātiwai tribe of eastern Northland.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="05cb692c634cd90e5411aab92ca3e649474ff786" id="img-2">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f5b3d20f0dc5c22a83f96fb709ecd204">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-2" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=adbbf5d870d9cc7f0b9a24eb5472ebf3 1760w"> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f0a6d8fa60b5571e9e0e9a5673e407b7 880w"> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d39897e20bb677fda3feb14113aad381 1600w"> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=dbf240ee147c5a0a43321f1634ee41eb 800w"> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f7926915234cc22c9fe718771f5837cc 1280w"> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=fa3edf920739ca39d41e3fce38ab277b 640w"> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f9fd7969943957bd4893fe29d248626c 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=61ab70443d54f672febc609b4bfbc5c0 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=a416bff5ba9e0d62f1634aee83308528 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5ae3b22ecb3c7bde8b49a212d52b707c 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c43d04fbea54b99991b541ce674da43d 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a0c813b07b8d5b99a33a7133ea7185db 445w">
<img itemprop="contentUrl" alt="Hori Parata at his Pātaua farm, the place where he was born and grew up." src="https://i.guim.co.uk/img/media/05cb692c634cd90e5411aab92ca3e649474ff786/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=575838a657b26493e956c7f84b058080"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Hori Parata at his Pātaua farm, the place where he was born and grew up
</p>
</li>
</ul>
<p>
“We’re having to put up with a lot of stuff today. The public want to hug the whales, they want to touch them, they want to feel good – that’s not the thing. We feel that is ridiculous.”
</p>
<p>
Whale experts regard New Zealand – or Aotearoa as it is called by Māori – as the whale stranding capital of the world, with more than 5,000 incidents recorded since 1840, and an average of 300 individual animals beaching themselves each year.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="98c683a7df9c83b2c13de2d93ca1825199ed5150" id="img-3">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2432083289db6bd42f6ad7adc64a78d">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3166"><a href="#img-3" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7234718ccc92b4251bdfe4c505f064f8 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b80f7e8fcfbbe8e12fa6954ddeb3bd1e 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=78ebc960d0c15f627bfbe60cd770a139 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1319684ba57c18162a45e56384d418b0 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3170b3a80d76d6c6422b65045444829e 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e835570164cc241d4009af47fc1051f7 445w">
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, watched by his father Hori Parata, carves a traditional Maōri design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata" src="https://i.guim.co.uk/img/media/98c683a7df9c83b2c13de2d93ca1825199ed5150/0_0_4800_3166/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2f198e1958f140f3ac664a3fdd87177c"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Kauri (Te Kaurinui Robert) Parata, watched by his father, Hori Parata, carves a traditional Māori design at their home in Whangārei. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father
</p>
</li>
</ul>
<p>
Concrete information on why whales strand remains elusive, but “sickness, navigational error, geographical features, a rapidly falling tide, being chased by a predator, or extreme weather” are all thought to contribute, according to the New Zealand Department of Conservation.
</p>
<p>
Climate change is to blame too, <a href="https://www.radionz.co.nz/news/national/377272/new-zealand-beached-whales-why-are-so-many-getting-stranded" data-link-name="in body link">scientists think</a>, with warming ocean temperatures moving whales’ prey closer to the shore and forcing them to pursue their food into shallow waters.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0447972cf47ca67882fcfc648edf7e574b0853bc" id="img-4">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0e7302276c6cc7cb65be9bcdacd3081f">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-4" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=933c5693fbf9195f6e83a8928283f85e 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e07b988894308f3bfc052d1ff9dfc1e2 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=113cf92fe80bc5996634c34b3c7a0c09 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=5a7358223e80941bd1b0e0f427beefde 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=292e941797de1671c185c1b074c688ad 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d1bc1c4d9341f9b6d63b64861a3de711 445w">
<img itemprop="contentUrl" alt="A bin of small whale bones." src="https://i.guim.co.uk/img/media/0447972cf47ca67882fcfc648edf7e574b0853bc/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=718132fd888108a18383c24a8425523b"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="416800b8d06039780c3e6de28564e6f277b4e7b7" id="img-5">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=249ddc3119aa637c3a1ae4998509f604">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-5" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4197b7a2bc3c7e32d3eaee927b9d08e6 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=82d6908c6ffe622566e1cc6513d60ddd 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=222e300fb2a60da1a841fbbc2bc8f752 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=63ba1cfe4b5e7c7d741d311d58997fcf 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=231139825b311ede8d01e6c702d6d12c 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1065dd4439e14c72fe520c19566172e2 445w">
<img itemprop="contentUrl" alt="The baleen recovered from a stranded Pygmy Right Whale." src="https://i.guim.co.uk/img/media/416800b8d06039780c3e6de28564e6f277b4e7b7/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c5a53178ebbe54490c97fad6b5e032c4"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="8c207197c0a9e6f407dcddfded5f868a142c9cab" id="img-6">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=bcfefc5593dcc2bdb0dd3b0543cfa4eb">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-6" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=d272c4dc57408daa0dcdf5461a230e9e 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=890f0e0cee54663a90292471a16f95d2 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=59a57c5bded93fe0efe0046226fe7c69 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f1b2a4c79e965aa76a322ad25072a052 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=ffbdfd671eb45e3a87e3dc9137e8b006 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c560f0b0ef9736c1a66215cc29c59d43 445w">
<img itemprop="contentUrl" alt="Squid beaks, from the stomach of a Sperm Whale." src="https://i.guim.co.uk/img/media/8c207197c0a9e6f407dcddfded5f868a142c9cab/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0dd659ae339b1aea9a99ed7f6f8eadeb"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Clockwise from top: small whale bones; squid beaks, from the stomach of a sperm whale; the baleen filter-feeder system recovered from a stranded pygmy right whale.
</p>
</li>
</ul>
<h2>
‘Unprecedented’ strandings
</h2>
<p>
November marked the beginning of whale stranding season, and it started with a surge in incidents, according to whale rescue group Project Jonah, with <a href="https://www.theguardian.com/environment/video/2018/nov/26/whales-die-stranded-stewart-island-new-zealand-beach-video" data-link-name="in body link">140 pilot whales</a> beaching and dying on Stewart Island, 10 rare pygmy whales on Ninety Mile beach, 51 stranded and dead on the Chatham Islands and a spate of individual cases around the country.
</p>
<p>
And as more whales beach and die – from exhaustion, heat stroke or seagulls feasting on their flesh – an acute sense of grief is growing among New Zealand’s indigenous people, who regard whales as their ancestors and <em>taonga</em> (treasures).
</p>
<p>
“These days it is like a zoo. People just want to come and gawk at us, without even trying to understand what is happening with the animals and the environment,” says Parata, bristling with anger.
</p>
<figure data-interactive="https://interactive.guim.co.uk/embed/iframe-wrapper/0.1/boot.js" data-canonical-url="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-alt="whale strandings">
<a href="https://interactive.guim.co.uk/uploader/embed/2019/01/stranded_whales/giv-3902U99iNUM3iDSZ/" data-link-name="in body link">whale strandings</a>
</figure>
<p>
“When will we talk about what is hurting these animals out on the sea? They are drowning out there, they can’t breathe, they beach themselves to be with the Aunties.”
</p>
<p>
Ngātiwai believe the whales beach when they are ready to die and want to return to their families, the Māori people. Then, their human families use the whales’ gift of their bodies for sacred carvings, for traditional medicines, and even for compost.
</p>
<p>
There are marked tribal differences across New Zealand and while some tribes work to refloat stranded whales, others like Parata’s Ngātiwai stand back and allow the Department of Conservation and volunteer groups to take the lead in rescue efforts.
</p>
<p>
Then the tribe moves in en masse and holds a <em>karakia</em> (prayer), names each animal and sets to work removing their bones, blubber, eyes and teeth for cultural purposes.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="0d13adeb0790af5c5fa317ce477c323d0e1c773c" id="img-7">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d73916fda0f6c60b5eebcb71fea643e9">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="2334"><a href="#img-7" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=524b92a6590a1b5c139bfe50581476a1 2600w"> <source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=837d7df48e565ffad49febab20fbd179 1300w"> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=fe98f4515ae3490f53804235a4dcf84b 2280w"> <source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a3ec73ab7f20798ec264f5ecb59f6bcc 1140w"> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=3ac3fd319accb248cf8a13e8ef55ebcc 2250w"> <source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2561c80af0d0e8eed2fc7c535be48696 1125w"> <source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=539617a30ea852b01218eec69d6066e6 1930w"> <source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=624139eae65dd4c96efd1a90243b9286 965w"> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=633619afab6a0ee26a3839b69160e854 1450w"> <source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98b9351340e8ee30063a4b6eb6501d6f 725w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e11a31f8adaab06b457fd9c7dda6bf6d 1290w"> <source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=912104c53b38d1cd424bfb82302480e1 645w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=209259fab8ec9e81386838f2d5cdecc3 930w"> <source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=06848a63793084a21bc86aa613b126a7 465w">
<img itemprop="contentUrl" alt="Buck Cullen with his daughter Kaiarahi (10 months) in his back yard where he is storing a pair of massive Sperm Whale jawbones. Buck is a integral member of the whale recovery team, alongside Hori Parata." src="https://i.guim.co.uk/img/media/0d13adeb0790af5c5fa317ce477c323d0e1c773c/0_0_4800_2334/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=41b88ee343b9be76688b88443c7a8958"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Buck Cullen with his daughter Kaiarahi (10 months) in his backyard, where he is storing a pair of massive sperm whale jawbones. Cullen is an integral member of Hori Parata’s whale recovery team
</p>
</li>
</ul>
<p>
But indigenous elders say they aren’t being listened to when they tell the government their whale kin are sick, and trying to escape an increasingly polluted and unpredictable ocean.
</p>
<p>
Earlier this year in South Taranaki, a mass stranding that was described as <a href="https://www.stuff.co.nz/national/104249829/unprecedented-whale-strandings-reaches-11-in-total-on-taranaki-beach" data-link-name="in body link">“unprecedented”</a> left the local Māori tribe scrambling. Security was brought in when thieves attacked a sperm whale with an axe, trying to remove valuable teeth from its jaw.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="4973b41f53b8ade499f99a305b01157eca659ca5" id="img-8">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ee921715aaf10e0816dcf15dfc3a21f5">
<meta itemprop="width" content="1200">
<meta itemprop="height" content="900"><a href="#img-8" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8eb5af62092c8a0eb0d283cb0887f5c5 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9ace0f595be568263c5991993632dae8 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=97219f131f1013d15fed7c63308f2b2f 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=17191691b6cd81c2a67730d5475db08b 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=30d51b75767551399ba174bae5c39e94 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=936568b6e7da0a710abcd2c015fd0a8b 445w">
<img itemprop="contentUrl" alt="12 Parāoa Whales (Sperm Whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen on their coast in recent memory." src="https://i.guim.co.uk/img/media/4973b41f53b8ade499f99a305b01157eca659ca5/0_0_1200_900/master/1200.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7c255bf6f8a27c56365a86813cdd1517"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
12 parāoa whales (sperm whales) recently stranded on the South Taranaki coast of Kaupokonui, on a scale not seen near this location in recent memory
</p>
</li>
</ul>
<p>
Parata and his 22-year-old son, Te Kaurinui Robert Parata, were called in to assist. Te Kaurinui was called after the first whale his father ever named, and left university this year to return to Whangārei and study whale <em>tikanga</em> (protocol) and carving.
</p>
<p>
He says mass strandings are getting more local and international attention and money from donations, but traditional knowledge is being dismissed as overly spiritual.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d1941b6a6908314fab28f44da222a4c892213341" id="img-9">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4e46b70079819caee86359a058c92be1">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3120"><a href="#img-9" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b775d649981f7bc36cf0753dd9f47862 1760w"> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3aa0de336dad5991ff92d488288fcc92 880w"> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9133cf8cd42d42bb1342dbefbfc67c22 1600w"> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2916efc01252a457768bfd28b40591ee 800w"> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=c7044b93d3aa73f7eac1b56ab9d77d86 1280w"> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2342e6519bfe1ce74a04ca96c06ed4cd 640w"> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=b6218129f26e09d8133bf226df3e9bed 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=9d20a1ca789f2efe7ba2dfdef9c28725 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=73fb4a4f8d59a92325fef9faa2ddf90d 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7894b6600fbb6af1ec6552007ad12da8 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=19022ce9fcafebe383880059f34add78 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ded05f2c1859d70d3b8d3de3dcb75e0f 445w">
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, of the New Zealand Māori tribe Ngāti Wai, in front of the carving shed at Hihiaua Cultural Centre in Whangarei" src="https://i.guim.co.uk/img/media/d1941b6a6908314fab28f44da222a4c892213341/0_0_4800_3120/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=76a26a289728e3d625e57d32eced57d8"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Clockwise from top: Te Kaurinui Parata, in front of the carving shed at Hihiaua Cultural Centre in Whangārei; Parata holds three whale teeth recovered from a beached whale – the middle one shows marks where a poacher had attempted to hack it out with an axe before the recovery group arrived; the Pou, a tribal identifier, in front of the carving shed.
</p>
</li>
</ul>
<h2>
‘We need to listen’
</h2>
<p>
Māori harvest rights over dead whales have only been officially recognised since 1998, and the practice still elicits horror from some New Zealanders and visitors.
</p>
<p>
“Our own ancestors wouldn’t say to go down there and hug the whales. That’s a modern thing,” says Te Kaurinui.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="14b462f5d9489def554e0f9f436f13aec332f7b8" id="img-10">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b8bf0c299908608310fb47e4163afb38">
<meta itemprop="width" content="4278">
<meta itemprop="height" content="4800"><a href="#img-10" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=1a392a17d0c74d513ff14129ae5f7e6c 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d5b05458659a4a06ff6583046f712ad1 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=649c2127cfbab69a7d6e4085ea098d95 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d45dfb664b5479650ba067c6a5af16c3 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f94f4c652479979124e52c237804eb83 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f6c8d3b38a8af6d197d0a60d22a326f3 445w">
<img itemprop="contentUrl" alt="The Pou in front of the carving shed at Hihiaua Cultural centre" src="https://i.guim.co.uk/img/media/14b462f5d9489def554e0f9f436f13aec332f7b8/0_0_4278_4800/master/4278.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=a45ec7578a392eec201d2f6920b609a0"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="e2cf54c36f17c6894844ea0cdd4346288a002da9" id="img-11">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=3691946929b4105ba35ff9d306cb4fbc">
<meta itemprop="width" content="3172">
<meta itemprop="height" content="3189"><a href="#img-11" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=be6cf9beea564e7be872193b01c329a3 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=c74497ec55bb1b680169d62684aa803d 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=06b2c03638556f85da582e419cf1817a 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8cc04403e3902dbe1e4a9b01b8d4e517 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=893cbf39d6db1d28ba8fc85419382f9d 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6df725aaf59e21b09434b967e05b3272 445w">
<img itemprop="contentUrl" alt="Kauri (Tekaurinui Robert) Parata, holds three whale teeth recovered from a beached whale. The middle tooth shows the marks where a poacher had attempted to hack it out with an axe before the recovery group arrived. Kauri is a member of the Manu Taupunga group that is the organising arm of the whale-body recovery operation started by his father, Hori Parata." src="https://i.guim.co.uk/img/media/e2cf54c36f17c6894844ea0cdd4346288a002da9/915_0_3172_3189/master/3172.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0bd8e9f51bdf79a6e0a15ed176cfb57d"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<p>
The Ngātiwai are investigating a possible link between the crisis of the dieback disease killing New Zealand’s native kauri trees – and <a href="https://www.theguardian.com/world/2018/jul/14/like-losing-family-time-may-be-running-out-for-new-zealands-most-sacred-tree" data-link-name="in body link">threatening the giant Tāne Mahuta, which may be 2,000 years old</a> – and the increase in whale strandings.
</p>
<p>
Parata and his family believe whale oil and byproducts could be used to try to cure Kauri dieback, and want more government money and attention directed towards indigenous knowledge of the interconnectedness of the New Zealand environment, and possible indigenous solutions.
</p>
<p>
“People dismiss us when we tell them our spiritual understanding of whales – why they are beaching, why they are hurting,” says Te Kaurinui.
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="b5f3736b2ba2ef4df364258b0efcaba26f571d6e" id="img-12">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=72146577e379bdf9a21bc47994de5fb6">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3073"><a href="#img-12" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=4bd713ae6f60b56e36378bb2fe45e9eb 2600w"> <source media="(min-width: 1300px)" sizes="1300px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=b93c6d33ca7496220eb30d0c8bfaccf9 1300w"> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=f66f5a256170eced3c0a52aee235ae29 2280w"> <source media="(min-width: 1140px)" sizes="1140px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1f60eb7cf5fc6f899c7c9af9c05df9a0 1140w"> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=27526c9caf1a62110dfa16214ceccf14 2250w"> <source media="(min-width: 980px)" sizes="1125px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=1125&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7d47ba3d7898a398e1f9d8f2d19d391e 1125w"> <source media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=8974cfeb3e33bc51b55518f1bc81e68d 1930w"> <source media="(min-width: 740px)" sizes="965px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=965&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=04f953b4ec9555178204a1621b8a1f59 965w"> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=7e2c1494173f83f555806120a1fe43cc 1450w"> <source media="(min-width: 660px)" sizes="725px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=725&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=8f53cf25b514e283cce3d995220ca19b 725w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2b770ca04273c921c8abe0a7de72030a 1290w"> <source media="(min-width: 480px)" sizes="645px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=645&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=0da775328d9ade5ca605079a9118a64a 645w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e2e51037c5a6b8bf22d87cb927e45888 930w"> <source media="(min-width: 0px)" sizes="465px" srcset="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=465&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7ecf73b514ceb1b5e0391f17c7a4d3b0 465w">
<img itemprop="contentUrl" alt="Whangārei Harbour from Tamaterau, looking south through Mangrove sprouts coming up through the harbourside silt." src="https://i.guim.co.uk/img/media/b5f3736b2ba2ef4df364258b0efcaba26f571d6e/0_0_4800_3073/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7fcadc35b3a44ebafe3c469c6e89241d"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Whangārei Harbour seen from Tamaterau, with mangrove sprouts coming up through the harbourside silt
</p>
</li>
</ul>
<p>
“We are not foreigners in this land. We did not take this land off anyone else. We were not lost waiting for some bullheads to tell us what was going on.”
</p>
<p>
Kaitaia conservation department ranger Jamie Werner of Ngātiwai recently attended his first mass beaching on Ninety Mile Beach. It was the first recorded time pygmy whales had stranded on New Zealand shores.
</p>
<p>
“I arrived at the beach and we leapfrogged between the animals. They were calling out to each other and reassuring each other,” says Werner. “It was a shock. We’re working to adapt but the ocean is changing so fast.”
</p>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="d5aaf60e3a427f278747acf0c3e7ba39b39ef923" id="img-13">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1d00d3ea91d32110b40d85ff43227728">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-13" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 1300px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1300px) and (min-resolution: 120dpi)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=2a64f318f02a31bdf9f17ae01fca72a3 1760w"> <source media="(min-width: 1300px)" sizes="880px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=880&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=811b3a2ecee1a87b72813d328f59aa85 880w"> <source media="(min-width: 1140px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 1140px) and (min-resolution: 120dpi)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=47418e3f630c00af6124c471fdc54c20 1600w"> <source media="(min-width: 1140px)" sizes="800px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=800&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=93ec03888887d5eb6eabf07e40f2ed3b 800w"> <source media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=476d165704890f40cc5f2d74f4d09b95 1280w"> <source media="(min-width: 980px)" sizes="640px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=640&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=6179da6bca697f83ac063a56660aa31b 640w"> <source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=df0c5b9ccac02dcd010d7f3ed8ff8c85 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=e9fd2eedc9bb80286a6cabd1c7c3b0e2 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=e48f277772be7e455f9a4a4139b3bf4c 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=2e0af911fcd7011f04ee91d445290e84 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=15161e07957f327b44aa124d94ae8291 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=ef6c6809f2adcff0f4ff32899095839b 445w">
<img itemprop="contentUrl" alt="The skull of a Brydes whale, in the storage container at Hihiaua Cultural Centre, Whangārei." src="https://i.guim.co.uk/img/media/d5aaf60e3a427f278747acf0c3e7ba39b39ef923/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=1b74b488aedb864287ff160f86d74c9d"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<ul>
<li>
<p>
Above, the skull of a bryde’s whale; right, a large-calibre bullet of the type that the New Zealand Department of Conservation uses for euthanasing stranded whales that are beyond rescue
</p>
</li>
</ul>
<figure itemprop="associatedMedia image" itemscope="itemscope" itemtype="http://schema.org/ImageObject" data-component="image" data-media-id="3766106f73e858d5b140ae3cdd2eef84060180cd" id="img-14">
<meta itemprop="url" content="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=700&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=7dca3d798aef3526f42e281771d50f91">
<meta itemprop="width" content="4800">
<meta itemprop="height" content="3200"><a href="#img-14" data-link-name="Launch Article Lightbox" data-is-ajax>
<div>
<picture>
<source media="(min-width: 660px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 660px) and (min-resolution: 120dpi)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0daf794415132fc19259f8d2f654f57f 1240w"> <source media="(min-width: 660px)" sizes="620px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=620&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=98ce62f9d983fe8059c936a6c6cdff33 620w"> <source media="(min-width: 480px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 480px) and (min-resolution: 120dpi)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=9ff4aac2a3b07867dfdfb8f470ea6977 1210w"> <source media="(min-width: 480px)" sizes="605px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=605&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=10d1b71094ecf1722f593eb49bb2effe 605w"> <source media="(min-width: 0px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 0px) and (min-resolution: 120dpi)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=0705690fcb523f2781a5952f83528ff9 890w"> <source media="(min-width: 0px)" sizes="445px" srcset="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=445&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=4c748337df5f0348eb0e7d3e3ec46571 445w">
<img itemprop="contentUrl" alt="A large calibre bullet of the type that the New Zealand Department of Conservation (DOC) uses for euthanasing stranded whales that are beyond rescue." src="https://i.guim.co.uk/img/media/3766106f73e858d5b140ae3cdd2eef84060180cd/0_0_4800_3200/master/4800.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=d2f5bb7c3c3642ac8733ca40509f6e20"></picture>
</div><span><svg width="22" height="22" viewBox="0 0 22 22">
<path d="M3.4 20.2L9 14.5 7.5 13l-5.7 5.6L1 14H0v7.5l.5.5H8v-1l-4.6-.8M18.7 1.9L13 7.6 14.4 9l5.7-5.7.5 4.7h1.2V.6l-.5-.5H14v1.2l4.7.6" /></svg></span></a>
</figure>
<p>
The recent spate of mass strandings has been described as “heartbreaking” by the conservation department.
</p>
<p>
But for Parata and his family the slow, painful deaths of their ancestors are personal – and ultimately devastating – for the health of the tribe and the sea.
</p>
<p>
“It’s very emotional. Our ancestors tell us the strandings are a sign from the sea. So what is the sea telling us? We need to listen.”
</p>
</div>

1369
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/guardian-1/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/js-link-replacement/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/js-link-replacement/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "abc",
"Image": null,
"Title": "Replace javascript: links",
"SiteName": null
}

7
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/js-link-replacement/expected.html vendored Normal file
View File

@ -0,0 +1,7 @@
<div>
<span>
<p>abc</p>
<p>def</p>
ghi
</span>
</div>

14
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/js-link-replacement/source.html vendored Normal file
View File

@ -0,0 +1,14 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Replace javascript: links</title>
</head>
<body>
<a href="javascript:">
<p>abc</p>
<p>def</p>
ghi
</a>
</body>
</html>

19
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/keep-tabular-data/expected-images.json vendored Normal file
View File

@ -0,0 +1,19 @@
[
"http:\/\/www.factorio.com\/static\/img\/factorio-wheel.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-277-finished-2.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-newly-finished.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-277-not-finished-2.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-library-grid-view.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-library-list-view.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-blueprint-book.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-the-hand.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-enemy-bases.gif",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-cliff-controls.gif",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-rectangles.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-moisture+aux-debug-map.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-moisture+aux-controls.gif",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-island.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-islands.png",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-accumulator.gif",
"https:\/\/cdn.factorio.com\/assets\/img\/blog\/fff-282-accumulator-comparison.png"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/keep-tabular-data/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, all posts",
"Image": "http:\/\/www.factorio.com\/static\/img\/factorio-wheel.png",
"Title": "Friday Facts #282 - 0.17 in sight | Factorio",
"SiteName": "Factorio.com"
}

524
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/keep-tabular-data/expected.html vendored Normal file
View File

@ -0,0 +1,524 @@
<div>
<p>Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, <a href="http://fakehost/blog/">all posts</a></p>
<h2>The release plan <span size="2">(kovarex)</span>
</h2>
<p>This week was the time to close and finish all the things that will go to 0.17.0.</p>
<p>Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.</p>
<p>
If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February.
</p>
<h3>After release plan</h3>
<p>
Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time.
</p>
<p>
The smaller releases will contain mainly:
</p>
<ul>
<li>Final looks and behaviour of new GUI screens as they will be finished.</li>
<li>New graphics.</li>
<li>New sounds and sound system tweaks.</li>
<li>Mini tutorial additions and tweaks.</li>
</ul>
<p>
This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates.
</p>
<h2>The GUI progress <span size="2">(kovarex)</span>
</h2>
<p>
There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional.
</p>
<table readabilityDataTable="1">
<tbody>
<tr>
<td></td>
<td>General&nbsp;UX</td>
<td>UX&nbsp;draft</td>
<td>UX&nbsp;review</td>
<td>UI&nbsp;mockup</td>
<td>UI&nbsp;review</td>
<td>Implementation draft</td>
<td>Implementation review</td>
<td>Final&nbsp;review</td>
</tr>
<tr>
<td>Load&nbsp;map</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Save&nbsp;map</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Graphics&nbsp;settings</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Control&nbsp;settings</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Sound&nbsp;settings</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Interface&nbsp;settings</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Other&nbsp;settings</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Map&nbsp;generator</td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Quick&nbsp;bar&nbsp;<i><b>Twinsen</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
</tr>
<tr>
<td>Train&nbsp;GUI&nbsp;<i><b>kovarex</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
</tr>
<tr>
<td>Technology&nbsp;GUI<i>&nbsp;<b>Oxyd</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
</tr>
<tr>
<td>Technology&nbsp;tooltip&nbsp;<i><b>Oxyd</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
</tr>
<tr>
<td>Blueprint&nbsp;library<i>&nbsp;<b>kovarex</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Shortcut&nbsp;bar&nbsp;<i><b>Oxyd</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Character&nbsp;screen&nbsp;<i><b>Dominik</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Help&nbsp;overlay&nbsp;<i><b>kovarex</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Manage/Install&nbsp;mods&nbsp;<i><b>Rseding</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Recipe/item/Entity&nbsp;tooltip&nbsp;<i><b>Twinsen</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Chat&nbsp;icon&nbsp;selector&nbsp;<i><b>?</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>New&nbsp;game&nbsp;<i><b>?</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Menu&nbsp;structure&nbsp;<i><b>?</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Main&nbsp;screen&nbsp;chat&nbsp;<i><b>?</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
<tr>
<td>Recipe&nbsp;explorer&nbsp;<i><b>?</b></i></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
<td><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px"></td>
</tr>
</tbody>
</table>
<p><i>* Newly finished things since the last update in <a href="https://www.factorio.com/blog/post/fff-277">FFF-277</a>.</i></p><h3>Blueprint library</h3>
<p>
The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in <a href="https://www.factorio.com/blog/post/fff-278">FFF-278</a>) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction.
</p>
<p>
So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in <a href="https://www.factorio.com/blog/post/fff-255">FFF-255</a>. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away.
</p>
<p>
In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks:
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-library-grid-view.png" width="900px">
</p>
<p>
We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-library-list-view.png" width="900px">
</p>
<p>
The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-book.png" width="900px">
</p>
<h3>The hand</h3>
<p>
Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case.
</p>
<p>
This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-the-hand.png" width="900px"><br>
<i>The hand is protecting the slot from the robots.</i>
</p>
<h2>Terrain generation updates <span size="2">(TOGoS)</span>
</h2>
<p>Everyone has different opinions about what makes a good Factorio world.
I've been working on several changes for 0.17, but the overarching theme
has been to make the map generator options screen more intuitive
and more powerful.</p>
<p>
This was talked about somewhat in an earlier FFF (<a href="https://www.factorio.com/blog/post/fff-258">FFF-258</a>) regarding ore placement,
but since then we found more stuff to fix.
</p>
<h3>Biter Bases</h3>
<p>
In 0.16, the size control for biter bases didn't have much effect.
The frequency control changed the frequency, but that also decreased the size of bases,
which <a href="https://forums.factorio.com/viewtopic.php?t=55113">wasn't generally what people wanted</a>.
</p>
<p>
For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-enemy-bases.gif">
<br><i>New preview UI showing the effects of enemy base controls.
In reality the preview takes a couple seconds to regenerate after every change,
but the regeneration part is skipped in this animation to clearly show the effects of the controls.</i>
</p>
<p>
If you don't like the relatively uniform-across-the-world placement of biters,
there are changes under the hood to make it easier for modders to do something different.
Placement is now based on <a href="https://wiki.factorio.com/Prototype/NamedNoiseExpression">NamedNoiseExpressions</a> "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity".
By overriding any of those, a modder could easily create a map where biters are found only at high elevations,
or only near water, or correlate enemy placement with that of resources, or any other thing
that can be expressed as a function of location.
</p>
<h3>Cliffs</h3>
<p>
We've added a 'continuity' control for cliffs. If you really
like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces.
Or you can turn it way down to make cliffs very rare or be completely absent.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-cliff-controls.gif">
<br><i>Changing cliff frequency and continuity. Since cliffs are based on elevation,
you'll have to turn frequency <strong>way</strong> up if you want lots of layers
even near the starting lake.</i>
</p>
<h3>Biome Debugging</h3>
<p>
Tile placement is based on a range of humidity and 'aux' values
(humidity and aux being properties that vary at different points across the world)
that are suitable for each type of tile. For example: grass is only placed in
places with relatively high humidity, and desert (not to be confused with plain old sand)
only gets placed where aux is high. We've taken to calling these constraints 'rectangles',
because when you plot each tile's home turf on a chart of humidity and aux,
they are shown as rectangles.
</p>
<p>
It's hard to make sense of the rectangles just by looking at the autoplace code
for each tile, so I wrote a script to chart them. This allowed us to ensure that
they were arranged as we wanted, with no gaps between them,
and with overlap in some cases.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-rectangles.png" width="900" height="900">
<br><i>Rectangles.</i>
</p>
<p>
Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story,
since tile placement also depends on a noise layer specific to each tile type,
and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up).
So to further help us visualize how humidity, aux, tile-specific noise, and
autoplace controls worked together to determine tiles on the map,
there are a couple of alternate humidity and aux generators that simply vary them
linearly from north-south and west-east, respectively.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-moisture+aux-debug-map.png" width="900" height="900">
<br><i>Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively.</i>
</p>
<p>
This map helped us realize that, rather than having controls
for each different type of tile, it made more sense to just
control moisture and aux (which is called 'terrain type' in the GUI,
because 'aux' doesn't mean anything).
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-moisture+aux-controls.gif" width="900" height="664">
<br><i>Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty.</i>
</p>
<p>
A pet project of mine has been to
put controls in the map generator GUI so that we could select generators
for various tile properties (temperature, aux, humidity, elevation, etc) at
map-creation time without necessarily needing to involve mods.
This was useful for debugging the biome rectangles, but my ulterior
motive was to, at least in cases where there are multiple options,
show the generator information to players. A couple of reasons for
this:
</p>
<ul>
<li>It was already possible for mods to override tile property generators via presets, but
we didn't have a place to show that information in the UI.
So switching between presets could change hidden variables in non-obvious ways
and lead to a lot of confusion.</li>
<li>I had dreams of shipping alternate elevation generators in the base game.</li>
</ul>
<h3>Water Placement</h3>
<p>
For 0.16 I <a href="https://www.factorio.com/blog/post/fff-207">attempted to make the shape of continents more interesting</a>. Some people <a href="https://www.reddit.com/r/factorio/comments/7n3mn1/016_swamplands_are_pure_awesome/">really liked</a> the <a href="https://www.youtube.com/watch?v=-pkwTjG-sJg">new terrain</a>, or at least <a href="https://www.reddit.com/r/factorio/comments/7krq86/question_about_water_generation_in_016/">managed to find some settings that made it work</a> for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days.
</p>
<p>
So in 0.17 we're restoring the <em>default</em> elevation generator to one very similar to that used
by 0.12. Which means large, sometimes-connected lakes.
</p>
<p>
The water 'frequency' control was confusing to a lot of people including myself.
It could be interpreted as "how much water", when the actual effect was to inversely
scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water.
So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage',
which do the same thing, but in a hopefully more obvious way.
Larger scale means larger land features, while more coverage means more of the map covered in water.
</p>
<h3>New Map Types</h3>
<p>
In order to ensure a decent starting area, the elevation generator
always makes a plateau there (so you'll never spawn in the middle of
the ocean), and a lake (so you can get a power plant running).
Depending on what's going on outside of that plateau, this sometimes resulted in
a circular ring of cliffs around the starting point,
which looked very artificial, and we wanted to reduce that effect.
</p>
<p>
In the process of solving that problem I created another custom generator for debugging purposes.
This one simply generated that starting area plateau in an endless ocean.
I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it
to illustrate the mechanics behind generating the starting area.
</p>
<p>
The rest of the team liked that setting so much that we're making it a player-selectable option.
So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12,
and 'Island', which gives you a single large-ish island at the starting point.
There's a slider to let you change the size of the island(s).
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-island.png" width="900" height="900">
</p>
<p>
Maps with multiple starting points will have multiple islands.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-islands.png" width="900" height="900">
<br><i>PvP islands!</i>
</p>
<p>
And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings)
to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options,
but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts
(e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider,
rather than being just vaguely related to it somehow),
we're switching to storing them as numbers.
This has the side-effect that if you don't mind
<a href="https://gist.github.com/Bilka2/a17841d0ff1028b63c60c2bbf6fa7e4f">editing some JSON</a>,
you'll be able to create maps with values outside the range provided by the GUI sliders.
</p>
<p>
Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum.
</p>
<h2>High-res accumulators <span size="2">(Ernestas, Albert)</span>
</h2>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-accumulator.gif">
</p>
<p>
The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was.
</p>
<p>
The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same.
</p>
<p>
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-accumulator-comparison.png" width="900px">
</p>
<p>
As always, let us know what you think on our <a href="https://forums.factorio.com/64912">forum</a>.
</p>
</div>

740
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/keep-tabular-data/source.html vendored Normal file
View File

@ -0,0 +1,740 @@
<html>
<head>
<title>Friday Facts #282 - 0.17 in sight | Factorio</title>
<meta property="og:title" content="Friday Facts #282 - 0.17 in sight | Factorio" />
<meta property="og:type" content="website" />
<meta property="og:image" content="http://www.factorio.com/static/img/factorio-wheel.png" />
<meta property="og:site_name" content="Factorio.com" />
<link href="/static/img/favicon.ico" rel="icon" type="image/x-icon" />
<link href="/static/img/favicon.ico" rel="shortcut icon" type="image/x-icon" />
<link href="/static/css/bootstrap.min.css" rel="stylesheet" type="text/css" />
<link href="/static/css/bootstrap-responsive.min.css" rel="stylesheet" type="text/css" />
<link href="/static/css/factorio.css" rel="stylesheet" type="text/css" />
<link href="/static/lightbox/css/lightbox.css" rel="stylesheet" type="text/css" />
<link href="/blog/rss" rel="alternate" title="Recent Blog Posts" type="application/atom+xml" />
<meta name="viewport" content="width=device-width" />
</head>
<body style="background: rgb(22, 22, 22) url(/static/img/stressed_linen_texture.png) repeat 0 0;">
<div class="container header" style="margin-top: 30px;">
<!-- top row -->
<div class="row" style="margin-bottom: 30px;">
<div class="span4">
<a href="/"><img src="/static/img/factorio-logo.png" style="margin-top: 10px;" /></a>
</div>
<div class="span8 span8-navbar" style="margin-top: 10px;">
<div class="user-controls">
<a href="/login">log in</a>
<a href="/signup">sign up</a>
</div>
<div class="navbar">
<ul class="nav">
<li>
<a href="/"> Home </a>
</li>
<li>
<a href="/store" style="color: #ff7200;"> Merch </a>
</li>
<li class="custom-dropdown">
<a class="custom-dropdown-toggle" data-toggle="dropdown" href="/game-overview" role="button">
Game
</a>
<ul class="custom-dropdown-menu" role="menu">
<li role="presentation">
<a href="/starter-page">
Starter Page
</a>
</li>
<li role="presentation">
<a href="/screenshots">
Screenshots
</a>
</li>
<li role="presentation">
<a href="/videos">
Videos
</a>
</li>
<li role="presentation">
<a href="/content">
Content
</a>
</li>
<li role="presentation">
<a href="/modding">
Modding
</a>
</li>
<li role="presentation">
<a href="/team">
Team
</a>
</li>
</ul>
</li>
<li class="custom-dropdown">
<a class="custom-dropdown-toggle" data-toggle="dropdown" href="/support-overview" role="button">
Support
</a>
<ul class="custom-dropdown-menu" role="menu">
<li role="presentation">
<a href="/help">
Help
</a>
</li>
<li role="presentation">
<a href="/faq">
FAQ
</a>
</li>
<li role="presentation">
<a href="/press-and-youtube">
Press and Youtube
</a>
</li>
<li role="presentation">
<a href="/community">
Community
</a>
</li>
<li role="presentation">
<a href="/partners">
Partners
</a>
</li>
<li role="presentation">
<a href="/credits">
Credits
</a>
</li>
</ul>
</li>
<li>
<a href="https://forums.factorio.com/"> Forums </a>
</li>
<li>
<a href="/contact"> Contact </a>
</li>
</ul>
</div>
</div>
</div>
</div>
<div class="container">
<noscript>
<div class="alert alert-block alert-error" style="font-weight: bold; ">
Javascript is required for proper functioning of these web pages.
</div>
</noscript>
<div class="row">
<div class="span12 blog-post">
<h2 style="margin-top: 20px; margin-bottom:5px;">
Friday Facts #282 - 0.17 in sight
</h2>
<div style="font-size: 11px;">Posted by kovarex, TOGos, Ernestas, Albert on 2019-02-15, <a href="/blog/">all posts</a></div>
<p>
</p>
<h2>The release plan <font size="2">(kovarex)</font>
</h2>
<p>This week was the time to close and finish all the things that will go to 0.17.0.</p>
<p>Not all of the things that we originally planned to be done were done (surprise), but we just left any non-essential stuff for later so we won't postpone the release any further. The plan is, that next week will be dedicated to the office playtesting and bugfixing. Many would argue, that we could just release instantly and let the players find the bugs for us, but we want to fix the most obvious problems in-house to avoid too many duplicate bug reports and chaos after the release. Also, some potential bugs, like save corruptions, are much more easily worked on in-house.</p>
<p>
If the playtesting goes well, we will let you know next Friday, and if it is the case, we will aim to release the week starting 25th February.
</p>
<h3>After release plan</h3>
<p>
Since there are a lot of things we would like to do before we can call 0.17 good enough, we will simply push new things into the 0.17 releases as time goes on. Even if 0.17 becomes stable in a reasonable time, we would still push things on top of it. We can still make experimental/stable version numbers inside 0.17. Most of the things shouldn't be big enough to make the game generally unstable. I've heard countless times a proposal to make small frequent releases of what have we added, this will probably be reality after 0.17 for some time.
</p>
<p>
The smaller releases will contain mainly:
</p>
<ul>
<li>Final looks and behaviour of new GUI screens as they will be finished.</li>
<li>New graphics.</li>
<li>New sounds and sound system tweaks.</li>
<li>Mini tutorial additions and tweaks.</li>
</ul>
<p></p>
<p>
This is actually quite a large change to our procedures, and there are many ways we will be trying to maximize the effectiveness of smaller and more regular content updates.
</p>
<h2>The GUI progress <font size="2">(kovarex)</font>
</h2>
<p>
There are several GUI screens that are finished. Others (most of them) are just left there as they are in 0.16. They are a combination of the new GUI styles and old ones. They sometimes look funny and out of place, but they should be functional.
</p>
<style>
.header_cell {
text-align:center;
font-weight: bold;
}
.finished {
text-align:center;
font-weight: bold;
}
.not_finished {
text-align:center;
font-weight: bold;
}
.finished_gui_table {
border-spacing: 10px;
border-collapse: collapse;
max-width: 900px;
}
.finished_gui_table td {
border: 1px;
border-style:solid;
padding: 5px;
}
</style>
<table class="finished_gui_table">
<tbody>
<tr>
<td></td>
<td class="header_cell">General&nbsp;UX</td>
<td class="header_cell">UX&nbsp;draft</td>
<td class="header_cell">UX&nbsp;review</td>
<td class="header_cell">UI&nbsp;mockup</td>
<td class="header_cell">UI&nbsp;review</td>
<td class="header_cell">Implementation draft</td>
<td class="header_cell">Implementation review</td>
<td class="header_cell">Final&nbsp;review</td>
</tr>
<tr>
<td>Load&nbsp;map</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Save&nbsp;map</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Graphics&nbsp;settings</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Control&nbsp;settings</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Sound&nbsp;settings</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Interface&nbsp;settings</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Other&nbsp;settings</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Map&nbsp;generator</td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Quick&nbsp;bar&nbsp;<i><b>Twinsen</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
</tr>
<tr>
<td>Train&nbsp;GUI&nbsp;<i><b>kovarex</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
</tr>
<tr>
<td>Technology&nbsp;GUI<i>&nbsp;<b>Oxyd</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
</tr>
<tr>
<td>Technology&nbsp;tooltip&nbsp;<i><b>Oxyd</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
</tr>
<tr>
<td>Blueprint&nbsp;library<i>&nbsp;<b>kovarex</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Shortcut&nbsp;bar&nbsp;<i><b>Oxyd</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Character&nbsp;screen&nbsp;<i><b>Dominik</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Help&nbsp;overlay&nbsp;<i><b>kovarex</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Manage/Install&nbsp;mods&nbsp;<i><b>Rseding</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Recipe/item/Entity&nbsp;tooltip&nbsp;<i><b>Twinsen</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Chat&nbsp;icon&nbsp;selector&nbsp;<i><b>?</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>New&nbsp;game&nbsp;<i><b>?</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Menu&nbsp;structure&nbsp;<i><b>?</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Main&nbsp;screen&nbsp;chat&nbsp;<i><b>?</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-282-newly-finished.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
<tr>
<td>Recipe&nbsp;explorer&nbsp;<i><b>?</b></i></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
<td class="not_finished"><img src="https://cdn.factorio.com/assets/img/blog/fff-277-not-finished-2.png" width="20px" /></td>
</tr>
</tbody>
</table>
<i>* Newly finished things since the last update in <a href="https://www.factorio.com/blog/post/fff-277">FFF-277</a>.</i>
<h3>Blueprint library</h3>
<p>
The blueprint library changes have been split into several steps. The reason is, that there was a big motivation to do the integration with the new quickbar (final version introduced in <a href="https://www.factorio.com/blog/post/fff-278">FFF-278</a>) in time for 0.17.0, while the other changes can be done after. The thing with the quickbar is, that it is quite a big change to one of the most used tools in the game and people generally don't like change even when it is for the better. To minimize the hate of the change, we need to "sell it properly". By that, we should provide as many of the positive aspects of the new quickbar at the time of its introduction.
</p>
<p>
So the change that is already implemented and working for 0.17 is the ability to put blueprints/books into the quick bar in a way that the quick bar is linked directly to the blueprint library window, so you don't need to have the physical blueprint items in your inventory. The other change is, that picking a blueprint from the blueprint library and then pressing Q will just dismiss it, instead of silently pushing it to your inventory. This works the same as the clipboard described in <a href="https://www.factorio.com/blog/post/fff-255">FFF-255</a>. You can still explicitly insert the blueprint from the library to an inventory slot, but if you just pick it, use it, and press Q, it goes away.
</p>
<p>
In addition to this, other changes related to the blueprint library will follow soon after 0.17.0. The first thing is the change of how the GUI looks:
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img style="vertical-align: top;" src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-library-grid-view.png" width="900px" />
</p>
<p>
We will also allow to switch between grid and list view. It mainly provides a way to nicely see the longer names of the blueprint. We noticed that players try to put a large amount of info about a blueprint in its name, so we are planning to add a possibility to write a textual description of the blueprint.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img style="vertical-align: top;" src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-library-list-view.png" width="900px" />
</p>
<p>
The last big change is to allow to put blueprint books into blueprint books, allowing better organisation. Basically like a directory structure. Whenever a blueprint/book is opened, we plan to show its current location, so the player knows exactly what is going on.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img style="vertical-align: top;" src="https://cdn.factorio.com/assets/img/blog/fff-282-blueprint-book.png" width="900px" />
</p>
<h3>The hand</h3>
<p>
Has it ever happened to you, that you have robots trying to put things into your full inventory, while you pick an item from it to build something, and then you just can't put it back, as the diligent robots just filled the last slot in your inventory by whatever they are trying to give to you? Wood from tree removal is the most frequent thing in my case.
</p>
<p>
This was annoying in 0.16 from time to time, but with the new quickbar, it started to happen even more, as now, you have only one inventory, and no reserved slots in the quickbar. To solve that, we just extended the "principal" of the hand. When you pick something from the inventory, the hand icon appears on the slot. As long as you hold the thing in your cursor, the hand stays there, and prevents other things from being inserted there. This way, you should always be able to return the currently selected item into your inventory as long as you didn't get it from external source like a chest.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img style="vertical-align: top;" src="https://cdn.factorio.com/assets/img/blog/fff-282-the-hand.png" width="900px" /><br />
<i>The hand is protecting the slot from the robots.</i>
</p>
<h2>Terrain generation updates <font size="2">(TOGoS)</font>
</h2>
<p>Everyone has different opinions about what makes a good Factorio world.
I've been working on several changes for 0.17, but the overarching theme
has been to make the map generator options screen more intuitive
and more powerful.</p>
<p>
This was talked about somewhat in an earlier FFF (<a href="https://www.factorio.com/blog/post/fff-258">FFF-258</a>) regarding ore placement,
but since then we found more stuff to fix.
</p>
<h3>Biter Bases</h3>
<p>
In 0.16, the size control for biter bases didn't have much effect.
The frequency control changed the frequency, but that also decreased the size of bases,
which <a href="https://forums.factorio.com/viewtopic.php?t=55113">wasn't generally what people wanted</a>.
</p>
<p>
For 0.17 we've reworked biter placement using a system similar to that with which we got resource placement under control. The size and frequency controls now act more like most people would expect, with frequency increasing the number of bases, and size changing the size of each base.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-enemy-bases.gif" />
<br /><i>New preview UI showing the effects of enemy base controls.
In reality the preview takes a couple seconds to regenerate after every change,
but the regeneration part is skipped in this animation to clearly show the effects of the controls.</i>
</p>
<p>
If you don't like the relatively uniform-across-the-world placement of biters,
there are changes under the hood to make it easier for modders to do something different.
Placement is now based on <a href="https://wiki.factorio.com/Prototype/NamedNoiseExpression">NamedNoiseExpressions</a> "enemy-base-frequency" and "enemy-base-radius", which in turn reference "enemy-base-intensity".
By overriding any of those, a modder could easily create a map where biters are found only at high elevations,
or only near water, or correlate enemy placement with that of resources, or any other thing
that can be expressed as a function of location.
</p>
<h3>Cliffs</h3>
<p>
We've added a 'continuity' control for cliffs. If you really
like mazes of cliffs, set it to high to reduce the number of gaps in cliff faces.
Or you can turn it way down to make cliffs very rare or be completely absent.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-cliff-controls.gif" />
<br /><i>Changing cliff frequency and continuity. Since cliffs are based on elevation,
you'll have to turn frequency <strong>way</strong> up if you want lots of layers
even near the starting lake.</i>
</p>
<h3>Biome Debugging</h3>
<p>
Tile placement is based on a range of humidity and 'aux' values
(humidity and aux being properties that vary at different points across the world)
that are suitable for each type of tile. For example: grass is only placed in
places with relatively high humidity, and desert (not to be confused with plain old sand)
only gets placed where aux is high. We've taken to calling these constraints 'rectangles',
because when you plot each tile's home turf on a chart of humidity and aux,
they are shown as rectangles.
</p>
<p>
It's hard to make sense of the rectangles just by looking at the autoplace code
for each tile, so I wrote a script to chart them. This allowed us to ensure that
they were arranged as we wanted, with no gaps between them,
and with overlap in some cases.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-rectangles.png" width="900" height="900" />
<br /><i>Rectangles.</i>
</p>
<p>
Having the humidity-aux-tile chart is all well and good, but doesn't tell the whole story,
since tile placement also depends on a noise layer specific to each tile type,
and could also influenced by user-adjustable autoplace controls (e.g. turning the grass slider up).
So to further help us visualize how humidity, aux, tile-specific noise, and
autoplace controls worked together to determine tiles on the map,
there are a couple of alternate humidity and aux generators that simply vary them
linearly from north-south and west-east, respectively.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-moisture+aux-debug-map.png" width="900" height="900" />
<br /><i>Using 'debug-moisture' and 'debug-aux' generators to drive moisture and aux, respectively.</i>
</p>
<p>
This map helped us realize that, rather than having controls
for each different type of tile, it made more sense to just
control moisture and aux (which is called 'terrain type' in the GUI,
because 'aux' doesn't mean anything).
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-moisture+aux-controls.gif" width="900" height="664" />
<br /><i>Sliding the moisture and aux bias sliders to make the world more or less grassy or red-deserty.</i>
</p>
<p>
A pet project of mine has been to
put controls in the map generator GUI so that we could select generators
for various tile properties (temperature, aux, humidity, elevation, etc) at
map-creation time without necessarily needing to involve mods.
This was useful for debugging the biome rectangles, but my ulterior
motive was to, at least in cases where there are multiple options,
show the generator information to players. A couple of reasons for
this:
</p>
<ul>
<li>It was already possible for mods to override tile property generators via presets, but
we didn't have a place to show that information in the UI.
So switching between presets could change hidden variables in non-obvious ways
and lead to a lot of confusion.</li>
<li>I had dreams of shipping alternate elevation generators in the base game.</li>
</ul>
<h3>Water Placement</h3>
<p>
For 0.16 I <a href="https://www.factorio.com/blog/post/fff-207">attempted to make the shape of continents more interesting</a>. Some people <a href="https://www.reddit.com/r/factorio/comments/7n3mn1/016_swamplands_are_pure_awesome/">really liked</a> the <a href="https://www.youtube.com/watch?v=-pkwTjG-sJg">new terrain</a>, or at least <a href="https://www.reddit.com/r/factorio/comments/7krq86/question_about_water_generation_in_016/">managed to find some settings that made it work</a> for them. Others called it a "swampy mess". A common refrain was that the world was more fun to explore in the 0.12 days.
</p>
<p>
So in 0.17 we're restoring the <em>default</em> elevation generator to one very similar to that used
by 0.12. Which means large, sometimes-connected lakes.
</p>
<p>
The water 'frequency' control was confusing to a lot of people including myself.
It could be interpreted as "how much water", when the actual effect was to inversely
scale both bodies of water and continents, such that higher water frequency actually meant smaller bodies of water.
So for 0.17, the water 'frequency' and 'size' sliders are being replaced with 'scale' and 'coverage',
which do the same thing, but in a hopefully more obvious way.
Larger scale means larger land features, while more coverage means more of the map covered in water.
</p>
<h3>New Map Types</h3>
<p>
In order to ensure a decent starting area, the elevation generator
always makes a plateau there (so you'll never spawn in the middle of
the ocean), and a lake (so you can get a power plant running).
Depending on what's going on outside of that plateau, this sometimes resulted in
a circular ring of cliffs around the starting point,
which looked very artificial, and we wanted to reduce that effect.
</p>
<p>
In the process of solving that problem I created another custom generator for debugging purposes.
This one simply generated that starting area plateau in an endless ocean.
I don't actually remember how this was useful for debugging, but at one point I directed Twinsen to look at it
to illustrate the mechanics behind generating the starting area.
</p>
<p>
The rest of the team liked that setting so much that we're making it a player-selectable option.
So in 0.17 you'll get to pick between the 'Normal' map type, which resembles that from 0.12,
and 'Island', which gives you a single large-ish island at the starting point.
There's a slider to let you change the size of the island(s).
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-island.png" width="900" height="900" />
</p>
<p>
Maps with multiple starting points will have multiple islands.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-islands.png" width="900" height="900" />
<br /><i>PvP islands!</i>
</p>
<p>
And speaking of scale sliders, we're expanding their range from ± a factor of 2 (the old 'very low' to 'very high' settings)
to ± a factor of 6 (shown as '17%' to '600%'). Previously the values were stored internally as one of 5 discrete options,
but as the recent terrain generation changes have made actual numeric multipliers more meaningful in most contexts
(e.g. the number of ore patches is directly proportional to the value of the 'frequency' slider,
rather than being just vaguely related to it somehow),
we're switching to storing them as numbers.
This has the side-effect that if you don't mind
<a href="https://gist.github.com/Bilka2/a17841d0ff1028b63c60c2bbf6fa7e4f">editing some JSON</a>,
you'll be able to create maps with values outside the range provided by the GUI sliders.
</p>
<p>
Mods will be able to add their own 'map types' to the map type drop-down, too. If you really liked the shape of landmasses in 0.16 and want to be able to continue creating new maps with it, please let us know on the forum.
</p>
<h2>High-res accumulators <font size="2">(Ernestas, Albert)</font>
</h2>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-accumulator.gif" />
</p>
<p>
The design of the accumulator has been always good. The 4 very visible cylinders, looking like giant batteries, Tesla poles and the electric beams perfectly telegraphed its function in terms of style and readability. That’s why for the high-res conversion we were very careful about keeping this entity as it was.
</p>
<p>
The only thing that was a bit disturbing (for some) are the poles crossing to each other when more than one accumulator is placed in a row. So we decided to fix it (or break it). The rest of the work was making the entity compatible for the actual look of the game. But in essence accumulators are still the same.
</p>
<p style="text-align: center; margin:auto; margin-top:20px; margin-bottom:20px;">
<img src="https://cdn.factorio.com/assets/img/blog/fff-282-accumulator-comparison.png" width="900px" />
</p>
<p>
As always, let us know what you think on our <a href="https://forums.factorio.com/64912">forum</a>.
</p>
<p></p>
<div class="footer">
<div class="footer-copyright">
Copyright © 2015 - 2019 Wube Software - all rights reserved.
</div>
<div class="footer-menu">
<a href="/terms-of-service" style="font-size: 11px;"> Terms of Service </a>
<span style="font-size: 11px;"> | </span>
<a href="/privacy-policy" style="font-size: 11px;"> Privacy </a>
<span style="font-size: 11px;"> | </span>
<a href="/imprint" style="font-size: 11px;"> Imprint </a>
<span style="font-size: 11px;"> | </span>
<a href="/presskit" style="font-size: 11px;"> Presskit </a>
<span style="font-size: 11px;"> | </span>
<a href="/faq" style="font-size: 11px;"> FAQ </a>
<span style="font-size: 11px;"> | </span>
<a href="/blog/rss" style="font-size: 11px;"> RSS </a>
<span style="font-size: 11px;"> | </span>
<a href="/jobs" style="font-size: 11px;">
Jobs
</a>
</div>
</div>
</div>
<script type="text/javascript" async="" src="https://ssl.google-analytics.com/ga.js"></script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js" type="text/javascript"></script>
<script src="/static/js/raphael-min.js" type="text/javascript"></script>
<script src="/static/js/factorio.min.js" type="text/javascript"></script>
<script src="/static/lightbox/js/lightbox.min.js" type="text/javascript"></script>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-115167276-1']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script');
ga.type = 'text/javascript';
ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0];
s.parentNode.insertBefore(ga, s);
})();
</script>
</div>
</div>
<div id="lightboxOverlay" class="lightboxOverlay" style="display: none;"></div>
<div id="lightbox" class="lightbox" style="display: none;">
<div class="lb-outerContainer">
<div class="lb-container"><img class="lb-image" src="data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==" />
<div class="lb-nav"><a class="lb-prev" href=""></a><a class="lb-next" href=""></a></div>
<div class="lb-loader"><a class="lb-cancel"></a></div>
</div>
</div>
<div class="lb-dataContainer">
<div class="lb-data">
<div class="lb-details"><span class="lb-caption"></span><span class="lb-number"></span></div>
<div class="lb-closeContainer"><a class="lb-close"></a></div>
</div>
</div>
</div>
</body>
</html>

12
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-1/expected-images.json vendored Normal file
View File

@ -0,0 +1,12 @@
[
"https:\/\/miro.medium.com\/max\/1200\/1*EO-pr4RolgcAOj_Uk1rpDA.png",
"https:\/\/miro.medium.com\/fit\/c\/96\/96\/1*vFTVh_mYyf0p6m7f77A3vw.jpeg",
"https:\/\/miro.medium.com\/max\/3788\/1*5o3M5niyi911waUrKWVZ0Q.png",
"https:\/\/miro.medium.com\/max\/1994\/1*8uOdeOfnUzTaFIY1r7oAMg.png",
"https:\/\/miro.medium.com\/max\/1698\/1*e7gjTlzi55udTXbbPeEs2A.png",
"https:\/\/miro.medium.com\/max\/1508\/1*JJkRh7JihTUo2apW_9ZXAQ.png",
"https:\/\/miro.medium.com\/max\/5760\/1*6wi5BlNNnykjZs0PufrvLQ.png",
"https:\/\/miro.medium.com\/max\/1694\/1*cS9IXYGfMmgxaAUlC7oqOQ.png",
"https:\/\/miro.medium.com\/max\/1860\/1*87KlGgfbuWP38nAaQaj3xw.png",
"https:\/\/miro.medium.com\/max\/1690\/1*kfOK60PtmWx6iP681-qRcg.png"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Vincent Vallet",
"Direction": null,
"Excerpt": "How to run a CPU profiling with Node.js on your production in real-time and without interruption of service.",
"Image": "https:\/\/miro.medium.com\/max\/1200\/1*EO-pr4RolgcAOj_Uk1rpDA.png",
"Title": "Node.js and CPU profiling on production (in real-time without downtime)",
"SiteName": "Voodoo Engineering"
}

343
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-1/expected.html vendored Normal file
View File

@ -0,0 +1,343 @@
<div>
<div>
<p><a rel="noopener" href="http://fakehost/@vincentvallet?source=post_page-----d6e62af173e2----------------------"><img alt="Vincent Vallet" src="https://miro.medium.com/fit/c/96/96/1*vFTVh_mYyf0p6m7f77A3vw.jpeg" width="48" height="48"></a>
</p>
</div>
<h2 id="0231">
Why CPU monitoring is important?
</h2>
<p id="d2c1">
I work at <a href="http://voodoo.io/" target="_blank" rel="noopener nofollow">Voodoo</a>, a French company that creates mobile video games. We have a lot of challenges with performance, availability, and scalability because of the insane amount of traffic our infrastructure supports (billions of events/requests per day …… no joke!). In this setting, every metric is important and gives us a lot of information about the state of our system.
</p>
<p id="0e89">
When working with Node.js one of the most critical resources to monitor is the CPU. Most of the time, when working on a low traffic API or project we don’t realize how many simple lines of code can have a huge impact on CPU. On the other hand, when traffic increases, a simple mistake can cost dearly.
</p>
<h2 id="292e">
Resources
</h2>
<p id="1efa">
What kind of resources does your application need? In most cases, we focus on memory and CPU. Good monitoring of these two elements is mandatory for an application running on production.
</p>
<p id="dce9">
For memory, constant monitoring is the best practice to track the worst developer nightmare a.k.a memory leak.
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/3788/1*5o3M5niyi911waUrKWVZ0Q.png" width="1894" height="970" role="presentation" data-old-src="https://miro.medium.com/max/60/1*5o3M5niyi911waUrKWVZ0Q.png?q=20">
</p>
</div>
<figcaption>
Memory leak in action
</figcaption>
</figure>
<p id="69dd">
A good way to debug memory leak is a memory dump and/or memory sampling but this is not the subject.
</p>
<p id="1fbc">
(for more details about V8 and its garbage collector you can read my previous article <a target="_blank" rel="noopener" href="http://fakehost/voodoo-engineering/nodejs-internals-v8-garbage-collector-a6eca82540ec">here</a>)
</p>
<blockquote>
<p>
Stay focused on the CPU!
</p>
</blockquote>
<p id="40e6">
Most of the time we monitor this resource with a simple solution allowing us to get a graph representing CPU consumption over time. If we want to be reactive we add an alarm, based on a threshold, to warn us when CPU usage is too high.
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1994/1*8uOdeOfnUzTaFIY1r7oAMg.png" width="997" height="230" role="presentation" data-old-src="https://miro.medium.com/max/60/1*8uOdeOfnUzTaFIY1r7oAMg.png?q=20">
</p>
</div>
<figcaption>
Basic CPU monitoring
</figcaption>
</figure>
<p id="0728">
And what next? We don’t have data about the state of the instance when the CPU usage has increased. So we can’t determine why we had this peak, at least not without an important time of debugging, comparing log, etc. This is exactly why you need to use CPU profiling.
</p>
<h2 id="8d00">
CPU profiling: what’s the difference with CPU monitoring?
</h2>
<blockquote>
<p>
“Most commonly, profiling information serves to aid program optimization. Profiling is achieved by instrumenting either the program source code or its binary executable form using a tool called a profiler”
</p>
</blockquote>
<p id="3e11">
Basically, for Node.js, CPU profiling is nothing more than collecting data about functions which are CPU consuming. And ideally, get a graphic representation of the collected data a.k.a “flame graph” or “flame chart”.
</p>
<p id="91c5">
It will help you to track the exact file, line, and function which takes the most time to execute.
</p>
<h2 id="088b">
What about existing solutions?
</h2>
<h2 id="dd40">
Add arguments to Node.js
</h2>
<p id="0306">
Node.js provides a way to collect data about CPU with two command lines.
</p>
<p id="66c8">
The first command just executes your application, the argument just tells to V8 engine to collect data. When you stop your script all information is stored in a file.
</p>
<pre><span id="16bd">node --prof app.js</span></pre>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1698/1*e7gjTlzi55udTXbbPeEs2A.png" width="849" height="534" role="presentation" data-old-src="https://miro.medium.com/max/60/1*e7gjTlzi55udTXbbPeEs2A.png?q=20">
</p>
</div>
<figcaption>
Output of — prof
</figcaption>
</figure>
<p id="57a6">
It is not very clear, is it?
</p>
<p id="abed">
That’s why you just need to run this second command to transform your raw file into a more human-readable output.
</p>
<pre><span id="061c">node --prof-process isolate-0xnnnnn-v8.log &gt; processed.txt</span></pre>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1508/1*JJkRh7JihTUo2apW_9ZXAQ.png" width="754" height="306" role="presentation" data-old-src="https://miro.medium.com/max/60/1*JJkRh7JihTUo2apW_9ZXAQ.png?q=20">
</p>
</div>
<figcaption>
The output of — prof-process
</figcaption>
</figure>
<p id="85fa">
It seems better, here you can determine which function consumes the most of CPU (percentage of the time).
</p>
<h2 id="9e54">
ClinicJs
</h2>
<p id="176a">
ClinicJs is a set of tools that allow you to collect data and display performance charts. With “clinic flame” you can generate a flame graph based on CPU consumption.
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/5760/1*6wi5BlNNnykjZs0PufrvLQ.png" width="2880" height="1534" role="presentation" data-old-src="https://miro.medium.com/max/60/1*6wi5BlNNnykjZs0PufrvLQ.png?q=20">
</p>
</div>
<figcaption>
Flame chart
</figcaption>
</figure>
<p id="5347">
But once again, you have to stop your app, launch the tool, then terminate the script in order to display the graph (files are generated on the disk).
</p>
<p id="d6e6">
For more details, you can see the <a href="https://clinicjs.org/" target="_blank" rel="noopener nofollow">project</a>.
</p>
<p id="be18">
<strong>To sum up</strong>, here is the list of drawbacks of the two previous solutions.
</p>
<ul class>
<li id="3bef">Downtime (you should kill your application to collect the data)
</li>
<li id="c0df">Performance overhead
</li>
<li id="27ec">Data collected locally
</li>
<li id="a4fd">Need external tools (ClinicJs)
</li>
</ul>
<p id="3f2c">
In conclusion: these are good solutions to debug on development environments and/or on a local machine.
</p>
<blockquote>
<p id="fcd9">
Unfortunately, CPU issues have a worrying tendency to occur on production, and when you are not in front of your screen.
</p>
</blockquote>
<h2 id="13ef">
Inspector
</h2>
<p id="294e">
“Inspector” refers to an API thanks to which you can debug your application. By debugging we mean to be able to connect directly to the core of Node.js to collect real-time data about the process.
</p>
<p id="ea23">
A module, available since version 8.x of Node.js, provides this kind of feature. There are two advantages to use it:
</p>
<ul class>
<li id="ed54">it’s native (no additional installation required)
</li>
<li id="7992">it can be used programmatically (no interruption)
</li>
</ul>
<p id="731f">
And here is how to make a CPU profiling with this module:
</p>
<figure>
</figure>
<p id="79d1">
As you can see, all the data is returned in variable “profile”. Basically, it’s a simple JSON object representing all the call stack and the CPU consumption for each function. And if you want to use an Async/await syntax you can install the module “inspector-api”.
</p>
<pre><span id="c085">npm install inspector-api --save</span></pre>
<p id="195d">
It also comes with a built-in exporter to send data to S3, with this method <strong>you don’t write anything on the disk</strong>!
</p>
<figure>
</figure>
<p id="964f">
If you use another storage system you can just collect the data and export it by yourself.
</p>
<figure>
</figure>
<h2 id="848b">
And now, CPU profiling on-demand!
</h2>
<p id="6933">
We have an API that we want to test with autocannon tool. At this step, our project is able to serve around 200 requests in 20 seconds. There is probably a mistake somewhere in the code which slows down our application.
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1694/1*cS9IXYGfMmgxaAUlC7oqOQ.png" width="847" height="362" role="presentation" data-old-src="https://miro.medium.com/max/60/1*cS9IXYGfMmgxaAUlC7oqOQ.png?q=20">
</p>
</div>
</figure>
<p id="fb78">
But now, what if we want to trigger a CPU profiling remotely (without ssh connection to the server)? It’s possible using Websocket, SSE or any other technology to send a message to your instance.
</p>
<p id="2c91">
Here is a simple example of a server using the “ws” module to send a message to a unique instance.
</p>
<figure>
</figure>
<p id="2206">
Of course, it only works with one instance, but it’s a fake project to demonstrate the principle ;)
</p>
<p id="e92d">
Now we can request our server to ask it to send a message to our instance and start/stop a CPU profiling. In your instance, you can handle the CPU profiling like this:
</p>
<figure>
</figure>
<p id="c3d0">
<strong>To sum up</strong>: we are able to trigger a CPU profiling, on-demand, in real-time, without interruption or connection to the server. Data can be collected on the disk (and extracted later) or can be sent to S3 (or any other system, PR are welcomed on the <a href="https://github.com/wallet77/v8-inspector-api" target="_blank" rel="noopener nofollow">inspector-api project</a>).
</p>
<blockquote>
<p id="6e87">
And because the profiler is a part of V8 itself, the format of the generated JSON file is compatible with the Chrome dev tools.
</p>
</blockquote>
</div><div>
<p id="2cda">
<strong>How can we identify an issue?</strong>
</p>
<p id="e0d2">
A CPU profiling should be read like this:
</p>
<ul class>
<li id="27e6">the x-axis shows the stack profile population
</li>
<li id="194a">the y-axis shows stack depth
</li>
</ul>
<p id="e950">
<strong>What does it mean?</strong>
</p>
<p id="174c">
The larger is a box (a function call) the more it consumed CPU. So a good CPU profiling should look like a “flame” graph where each stack is the finest possible.
</p>
<p id="48d9">
In our example, every request try to generate a token. For this purpose, it calls the function pbkdf2 which is CPU consuming. Our CPU profile looks like a sequence of big blocks of time, like if the last function in the call stack takes 99% of the total time.
</p>
<p id="d62c">
The CPU profiling after optimizations, with the same time range.
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1860/1*87KlGgfbuWP38nAaQaj3xw.png" width="930" height="523" role="presentation" data-old-src="https://miro.medium.com/max/60/1*87KlGgfbuWP38nAaQaj3xw.png?q=20">
</p>
</div>
<figcaption>
CPU profiling after optimizations
</figcaption>
</figure>
<p id="10ee">
As you can notice, we have to zoom to the profile if we want to see the call stack, because after optimizations the API was able to take a lot more traffic. Now every function in the call stack looks like a microtask.
</p>
</div><div>
<p id="10f1">
And now our application is able to serve more than 200,000 requests in 20 seconds; <strong>we increased the performance by a factor of 100k</strong>!
</p>
<figure>
<div>
<p><img src="https://miro.medium.com/max/1690/1*kfOK60PtmWx6iP681-qRcg.png" width="845" height="362" role="presentation" data-old-src="https://miro.medium.com/max/60/1*kfOK60PtmWx6iP681-qRcg.png?q=20">
</p>
</div>
</figure>
<h2 id="98b9">
More than just CPU profiling
</h2>
<p id="e1ad">
With the inspector module, you can do much more than just CPU profiling, here is a non-exhaustive list:
</p>
<ul class>
<li id="eb04">memory dump &amp; memory sampling
</li>
<li id="a9ea">code coverage
</li>
<li id="b896">use the debugger in real-time
</li>
</ul>
<h2 id="06d2">
Warnings
</h2>
<p id="731b">
Every tool, even the most powerful, comes with its own disadvantages. If you enable the profiler and/or the debugger on your production you have to keep an eye on two things:
</p>
<p id="e485">
<strong>1) performance overhead</strong>
</p>
<p id="0513">
A profiler needs to use CPU to work and it collects data into memory. The longer you let it run and the more CPU / memory it will need. This is why you should begin with very short CPU profiling, no more than a few seconds between the start and stop command. And never forget to monitor the impact of the profiler on your own infrastructure. If everything is fine you can increase the time and the frequency of CPU profiling.
</p>
<p id="049c">
One more very important thing: <strong>never forget to always stop a started CPU profiling</strong>. You can add a timer to automatically call the stop function after a while.
</p>
<p id="0656">
<strong>2) security</strong>
</p>
<p id="7999">
Using the inspector in Node.js it’s like opening the door of the core of your application. You should be very careful about who can use features like CPU profiling and/or the debugger. Never make the inspector “public” as being able to launch a feature from an unsafe route (not protected with an authentification mechanism). Even the collected data can be seen as critical, never send it to a system you do not trust.
</p>
<h2 id="5618">
Conclusion
</h2>
<p id="ae1a">
CPU profiling is really a must-have tool for every developer. And now, with some precautions, we can run it on production thanks to the amazing work done by the V8 and Node.js team.
</p>
<p id="1eab">
The inspector module offers a lot more features than you can use to debug your application.
</p>
<p id="0aba">
I will write another article about using CPU profiling and the inspector on production on a high traffic project.
</p>
<h2 id="3c5b">
Sources &amp; links
</h2>
<ul class>
<li id="d86d">
<a href="https://nodejs.org/api/inspector.html" target="_blank" rel="noopener nofollow">https://nodejs.org/api/inspector.html</a>
</li>
<li id="cc52">
<a href="https://chromedevtools.github.io/devtools-protocol/v8" target="_blank" rel="noopener nofollow">https://chromedevtools.github.io/devtools-protocol/v8</a>
</li>
<li id="d331">
<a target="_blank" rel="noopener" href="http://fakehost/netflix-techblog/node-js-in-flames-ddd073803aa4">https://medium.com/netflix-techblog/node-js-in-flames-ddd073803aa4</a>
</li>
<li id="6420">
<a href="https://www.npmjs.com/package/inspector-api" target="_blank" rel="noopener nofollow">https://www.npmjs.com/package/inspector-api</a>
</li>
</ul>
</div>

1277
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-1/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

3
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-2/expected-images.json vendored Normal file
View File

@ -0,0 +1,3 @@
[
"https:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/c_fill,f_auto,fl_progressive,g_center,h_675,pg_1,q_80,w_1200\/18zu12g5xzyxojpg.jpg"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-2/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Mama Robotnik",
"Direction": null,
"Excerpt": "Nothing beats the passion of a true fan writing about something they love. That's what you're about to see here: one of the richest, most amazing tributes to a great gaming series that we've ever run on Kotaku. Warning #1: this one might make your browser chug, so close your other tabs. Warning #2: This piece might make it hurt a little more than there are no new Metroid games from Nintendo on the horizon.",
"Image": "https:\/\/i.kinja-img.com\/gawker-media\/image\/upload\/c_fill,f_auto,fl_progressive,g_center,h_675,pg_1,q_80,w_1200\/18zu12g5xzyxojpg.jpg",
"Title": "The Spectacular Story Of Metroid, One Of Gaming's Richest Universes",
"SiteName": "Kotaku"
}

1108
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-2/expected.html vendored Normal file
View File

File diff suppressed because it is too large Load Diff

3868
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-2/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

4
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-3/expected-images.json vendored Normal file
View File

@ -0,0 +1,4 @@
[
"https:\/\/p3-juejin.byteimg.com\/tos-cn-i-k3u1fbpfcp\/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.jpg",
"https:\/\/p3-juejin.byteimg.com\/tos-cn-i-k3u1fbpfcp\/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.png"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-3/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": null,
"Image": null,
"Title": "Lazy Load with Alt includes jpg\/png\/webp extensions",
"SiteName": null
}

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-3/expected.html vendored Normal file
View File

@ -0,0 +1,8 @@
<div>
<article>
<h2>Test Case 1</h2>
<img data-src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.jpg" alt="performance.jpg" src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.jpg">
<h2>Test Case 2</h2>
<img data-src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.png" alt="performance.jpg" src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.png">
</article>
</div>

21
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/lazy-image-3/source.html vendored Normal file
View File

@ -0,0 +1,21 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Lazy Load with Alt includes jpg/png/webp extensions</title>
</head>
<body>
<article class="markdown-body">
<h2>Test Case 1</h2>
<img class="lazyload inited loaded"
data-src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.jpg"
alt="performance.jpg"
/>
<h2>Test Case 2</h2>
<img class="lazyload inited loaded"
data-src="https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/0579d17015b145a88dd93992c6447d7d~tplv-k3u1fbpfcp-watermark.png"
alt="performance.jpg"
/>
</article>
</body>
</html>

4
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/medicalnewstoday/expected-images.json vendored Normal file
View File

@ -0,0 +1,4 @@
[
"https:\/\/cdn1.medicalnewstoday.com\/content\/images\/headlines\/318\/318674\/hand-holding-brain-lightbulb.jpg",
"https:\/\/cdn1.medicalnewstoday.com\/content\/images\/articles\/318\/318674\/hand-holding-brain-lightbulb.jpg"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/medicalnewstoday/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "By Ana Sandoiu",
"Direction": null,
"Excerpt": "New research investigates the neurobiological timing of the so-called a-ha! moment that occurs we have come up with the solution to a complex problem.",
"Image": "https:\/\/cdn1.medicalnewstoday.com\/content\/images\/headlines\/318\/318674\/hand-holding-brain-lightbulb.jpg",
"Title": "How does the brain turn unconscious information into conscious thought?",
"SiteName": "Medical News Today"
}

102
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/medicalnewstoday/expected.html vendored Normal file
View File

@ -0,0 +1,102 @@
<div itemprop="articleBody">
<header>
Neuroscience tells us that most of the work done by our brains happens on an unconscious level, but when does that "a-ha!" moment occur? And what happens during it? New research investigates.
</header>
<p><img data-src="https://cdn1.medicalnewstoday.com/content/images/articles/318/318674/hand-holding-brain-lightbulb.jpg" alt="hand holding brain lightbulb" src="https://cdn1.medicalnewstoday.com/content/images/articles/318/318674/hand-holding-brain-lightbulb.jpg"><br>
<em>A new study investigates when the 'a-ha!' moment takes place in the brain, and how similar it is to other brain processes.</em>
</p>
<p>
Many of us have noticed that we seem to get our best ideas when we're in the shower, or that we can find the answer to a difficult question when we least think about it.
</p>
<p>
A large body of neuroscientific <a href="http://journals.sagepub.com/doi/abs/10.1177/0956797612446024" target="_blank" rel="noopener">studies</a> has pointed out that the brain does a lot of work in its spare time, the so-called idle state - wherein the brain does not appear to be thinking about anything at all - and that this is the time when it works at its hardest to find solutions to complex problems.
</p>
<p>
With time and advances in <a href="http://fakehost/articles/248680.php" title="What is neuroscience?">neuroscience</a>, it has become more and more clear to researchers that Freud <em>was</em> right and the mind, as well as the brain, do work unconsciously. In fact, it would be safe to say that what is consciously known to us is just the tip of a much larger iceberg, deeply submerged in unconscious waters.
</p>
<p>
But the exact moment at which information becomes known to us - or when the "tip of the iceberg" pierces through the water, and the unconscious becomes conscious - has been somewhat of a mystery, from a neuroscientific point of view.
</p>
<p>
In other words, we do not yet know when that intellectually satisfying "a-ha!" moment takes place, or what the biology is behind it. This is why a team of researchers at Columbia University in New York City, NY, set out to investigate this moment in more detail.
</p>
<p>
The scientists were led by Michael Shadlen, Ph.D., of Columbia University's Mortimer B. Zuckerman Mind Brain Behavior Institute, and the <a href="http://www.cell.com/current-biology/fulltext/S0960-9822(17)30784-4" target="_blank" rel="noopener">findings</a> were published in the journal <em>Current Biology.</em>
</p>
<h2>
The hypothesis
</h2>
<p>
Dr. Shadlen and colleagues started out from an interesting hypothesis, one which they derived from previous research on the neurobiological processes involved in decision-making.
</p>
<p>
As the authors explain, research conducted in both monkeys and humans shows that many of our decisions take place at a point when the brain "feels" as though it has gathered enough information, or when a critical level of information has been accumulated.
</p>
<p>
This process of making a decision once the brain has accumulated enough evidence bears the name of "bounded evidence accumulation." Reaching this threshold is important because, although the brain does not use <em>all</em> of the information available, it uses as much as is necessary to make a speedy yet accurate decision.
</p>
<p>
<strong>The researchers wondered whether or not this threshold is also responsible for our "eureka!" moments.</strong>
</p>
<p>
In Dr. Shadlen's words, "Could the moment when the brain believes it has accumulated enough evidence be tied to the person's awareness of having decided - that important 'a-ha!' moment?"
</p>
<h2>
Examining the 'a-ha!' moment
</h2>
<p>
To answer this question, the scientists asked five people to perform a "direction discrimination" task. In it, the participants looked at dots on a computer screen. The dots moved randomly, as grains of sand would when blown by the wind. The participants were asked to say in which direction the dots had moved.
</p>
<p>
The moment they "decided" which direction the dots seemed to be taking was considered to be the equivalent of the "a-ha!" moment.
</p>
<p>
In the center of the screen, there was a fixed point and a clock. The display also had two "choice targets" - namely, left or right - and these were the directions in which the participants had to decide that the dots had moved.
</p>
<p>
Shortly after the dots had stopped moving, the participants used an electronic, hand-held stylus to move the cursor in the direction that they thought the dots had moved.
</p>
<p>
To determine when the decision was made, the researchers used the technique called "mental chronometry" - that is, after they made their decision, the participants were asked to move the clock backward to the point when they felt that they had consciously done so.
</p>
<p>
"The moment in time indicated by the participants - this mental chronometry - was entirely subjective; it relied solely on their own estimation of how long it took them to make that decision," Dr. Shadlen says. "And because it was purely subjective, in principle it ought to be unverifiable."
</p>
<h2>
'A-ha' moment similar to making a decision
</h2>
<p>
However, by applying a mathematical model, the scientists were able to match these subjective decision times to the bounded evidence accumulation process.
</p>
<p>
<strong>The subjective decision times fit so well with what the scientists determined as the evidence accumulation threshold that they were able to predict the choices of four of the five participants.</strong>
</p>
<p>
"If the time reported to us by the participants was valid, we reasoned that it might be possible to predict the accuracy of the decision," explains Dr. Shadlen.
</p>
<p>
"We incorporated a kind of mathematical trick, based on earlier studies, which showed that the speed and accuracy of decisions were tied together by the same brain function." This "mathematical trick" was the evidence accumulation model.
</p>
<blockquote>
<p>
<span>"</span>Essentially, the act of becoming consciously aware of a decision conforms to the same process that the brain goes through to complete a decision, even a simple one - such as whether to turn left or right."
</p>
<p>
Michael Shadlen, Ph.D.
</p>
</blockquote>
<p>
In other words, the study shows that the conscious awareness of the "a-ha!" moment takes place precisely when the brain has reached that threshold of evidence accumulation.
</p>
<p>
The findings provide unique insights into the biology of consciousness, say the researchers, and they bring us closer to understanding the biological basis of decisions, ethics, and, generally, the human mind.
</p>
</div>

2012
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/medicalnewstoday/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/mercurial/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/mercurial/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Contents",
"Image": null,
"Title": "Shared Mutable History \u2014 evolve extension for Mercurial",
"SiteName": null
}

738
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/mercurial/expected.html vendored Normal file
View File

@ -0,0 +1,738 @@
<div id="evolve-shared-mutable-history">
<div id="contents">
<p>
Contents
</p>
<ul>
<li>
<a href="#evolve-shared-mutable-history" id="id4">Evolve: Shared Mutable History</a>
<ul>
<li>
<a href="#sharing-with-a-single-developer" id="id5">Sharing with a single developer</a>
<ul>
<li>
<a href="#publishing-and-non-publishing-repositories" id="id6">Publishing and non-publishing repositories</a>
</li>
<li>
<a href="#setting-up" id="id7">Setting up</a>
</li>
<li>
<a href="#example-1-amend-a-shared-changeset" id="id8">Example 1: Amend a shared changeset</a>
</li>
<li>
<a href="#example-2-amend-again-locally" id="id9">Example 2: Amend again, locally</a>
</li>
</ul>
</li>
<li>
<a href="#sharing-with-multiple-developers-code-review" id="id10">Sharing with multiple developers: code review</a>
<ul>
<li>
<a href="#id2" id="id11">Setting up</a>
</li>
<li>
<a href="#example-3-alice-commits-and-amends-a-draft-fix" id="id12">Example 3: Alice commits and amends a draft fix</a>
</li>
<li>
<a href="#example-4-bob-implements-and-publishes-a-new-feature" id="id13">Example 4: Bob implements and publishes a new feature</a>
</li>
<li>
<a href="#example-5-alice-integrates-and-publishes" id="id14">Example 5: Alice integrates and publishes</a>
</li>
</ul>
</li>
<li>
<a href="#getting-into-trouble-with-shared-mutable-history" id="id15">Getting into trouble with shared mutable history</a>
<ul>
<li>
<a href="#id3" id="id16">Setting up</a>
</li>
<li>
<a href="#example-6-divergent-changesets" id="id17">Example 6: Divergent changesets</a>
</li>
<li>
<a href="#phase-divergence-when-a-rewritten-changeset-is-made-public" id="id18">Phase-divergence: when a rewritten changeset is made public</a>
</li>
</ul>
</li>
<li>
<a href="#conclusion" id="id19">Conclusion</a>
</li>
</ul>
</li>
</ul>
</div>
<p>
Once you have mastered the art of mutable history in a single repository (see the <a href="http://fakehost/test/user-guide.html">user guide</a>), you can move up to the next level: <em>shared</em> mutable history. <tt><span>evolve</span></tt> lets you push and pull draft changesets between repositories along with their obsolescence markers. This opens up a number of interesting possibilities.
</p>
<p>
The simplest scenario is a single developer working across two computers. Say you’re working on code that must be tested on a remote test server, probably in a rack somewhere, only accessible by SSH, and running an “enterprise-grade” (out-of-date) OS. But you probably prefer to write code locally: everything is setup the way you like it, and you can use your preferred editor, IDE, merge/diff tools, etc.
</p>
<p>
Traditionally, your options are limited: either
</p>
<blockquote>
<div>
<ul>
<li>(ab)use your source control system by committing half-working code in order to get it onto the remote test server, or
</li>
<li>go behind source control’s back by using <tt><span>rsync</span></tt> (or similar) to transfer your code back-and-forth until it is ready to commit
</li>
</ul>
</div>
</blockquote>
<p>
The former is less bad with distributed version control systems like Mercurial, but it’s still far from ideal. (One important version control “best practice” is that every commit should make things just a little bit better, i.e. you should never commit code that is worse than what came before.) The latter, avoiding version control entirely, means that you’re walking a tightrope without a safety net. One accidental <tt><span>rsync</span></tt> in the wrong direction could destroy hours of work.
</p>
<p>
Using Mercurial with <tt><span>evolve</span></tt> to share mutable history solves these problems. As with single-repository <tt><span>evolve</span></tt>, you can commit whenever the code is demonstrably better, even if all the tests aren’t passing yet—just <tt><span>hg</span> <span>amend</span></tt> when they are. And you can transfer those half-baked changesets between repositories to try things out on your test server before anything is carved in stone.
</p>
<p>
A less common scenario is multiple developers sharing mutable history, typically for code review. We’ll cover this scenario later. First, we will cover single-user sharing.
</p>
<div id="sharing-with-a-single-developer">
<h2>
<a href="#id5">Sharing with a single developer</a><a href="#sharing-with-a-single-developer" title="Permalink to this headline"></a>
</h2>
<div id="publishing-and-non-publishing-repositories">
<h3>
<a href="#id6">Publishing and non-publishing repositories</a><a href="#publishing-and-non-publishing-repositories" title="Permalink to this headline"></a>
</h3>
<p>
The key to shared mutable history is to keep your changesets in <em>draft</em> phase as you pass them around. Recall that by default, <tt><span>hg</span> <span>push</span></tt> promotes changesets from <em>draft</em> to <em>public</em>, and public changesets are immutable. You can change this behaviour by reconfiguring the <em>remote</em> repository so that it is non-publishing. (Short version: set <tt><span>phases.publish</span></tt> to <tt><span>false</span></tt>. Long version follows.)
</p>
</div>
<div id="setting-up">
<h3>
<a href="#id7">Setting up</a><a href="#setting-up" title="Permalink to this headline"></a>
</h3>
<p>
We’ll work through an example with three local repositories, although in the real world they’d most likely be on three different computers. First, the <tt><span>public</span></tt> repository is where tested, polished changesets live, and it is where you synchronize with the rest of your team.
</p>
<p>
We’ll need two clones where work gets done, <tt><span>test-repo</span></tt> and <tt><span>dev-repo</span></tt>:
</p>
<div>
<pre>$ hg clone public test-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone test-repo dev-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
<p>
<tt><span>dev-repo</span></tt> is your local machine, with GUI merge tools and IDEs and everything configured just the way you like it. <tt><span>test-repo</span></tt> is the test server in a rack somewhere behind SSH. So for the most part, we’ll develop in <tt><span>dev-repo</span></tt>, push to <tt><span>test-repo</span></tt>, test and polish there, and push to <tt><span>public</span></tt>.
</p>
<p>
The key to shared mutable history is to make the target repository, in this case <tt><span>test-repo</span></tt>, non-publishing. And, of course, we have to enable the <tt><span>evolve</span></tt> extension in both <tt><span>test-repo</span></tt> and <tt><span>dev-repo</span></tt>.
</p>
<p>
First, edit the configuration for <tt><span>test-repo</span></tt>:
</p>
<div>
<pre>$ hg -R test-repo config --edit --local
</pre>
</div>
<p>
and add
</p>
<div>
<pre>[phases]
publish = false
[extensions]
evolve =
</pre>
</div>
<p>
Then edit the configuration for <tt><span>dev-repo</span></tt>:
</p>
<div>
<pre>$ hg -R dev-repo config --edit --local
</pre>
</div>
<p>
and add
</p>
<p>
Keep in mind that in real life, these repositories would probably be on separate computers, so you’d have to login to each one to configure each repository.
</p>
<p>
To start things off, let’s make one public, immutable changeset:
</p>
<div>
<pre>$ cd test-repo
$ echo 'my new project' &gt; file1
$ hg add file1
$ hg commit -m 'create new project'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
<p>
and pull that into the development repository:
</p>
<div>
<pre>$ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<div id="example-2-amend-again-locally">
<h3>
<a href="#id9">Example 2: Amend again, locally</a><a href="#example-2-amend-again-locally" title="Permalink to this headline"></a>
</h3>
<p>
This process can repeat. Perhaps you figure out a more elegant fix to the bug, and want to mutate history so nobody ever knows you had a less-than-perfect idea. We’ll implement it locally in <tt><span>dev-repo</span></tt> and push to <tt><span>test-repo</span></tt>:
</p>
<div>
<pre>$ echo 'Fix, fix, and fix.' &gt; file1
$ hg amend
$ hg push
</pre>
</div>
<p>
This time around, the temporary amend commit is in <tt><span>dev-repo</span></tt>, and it is not transferred to <tt><span>test-repo</span></tt>—the same as before, just in the opposite direction. Figure 4 shows the two repositories after amending in <tt><span>dev-repo</span></tt> and pushing to <tt><span>test-repo</span></tt>.
</p>
<blockquote>
<p>
[figure SG04: each repo has one temporary amend commit, but they’re different in each one]
</p>
</blockquote>
<p>
Let’s hop over to <tt><span>test-repo</span></tt> to test the more elegant fix:
</p>
<div>
<pre>$ cd ../test-repo
$ hg update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
<p>
This time, all the tests pass, so no further amending is required. This bug fix is finished, so we push it to the public repository:
</p>
<div>
<pre>$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
<p>
Note that only one changeset—the final version, after two amendments—was actually pushed. Again, Mercurial doesn’t transfer hidden changesets on push and pull.
</p>
<p>
So the picture in <tt><span>public</span></tt> is much simpler than in either <tt><span>dev-repo</span></tt> or <tt><span>test-repo</span></tt>. Neither of our missteps nor our amendments are publicly visible, just the final, beautifully polished changeset:
</p>
<blockquote>
<p>
[figure SG05: public repo with rev 0:0dc9, 1:de61, both public]
</p>
</blockquote>
<p>
There is one important step left to do. Because we pushed from <tt><span>test-repo</span></tt> to <tt><span>public</span></tt>, the pushed changeset is in <em>public</em> phase in those two repositories. But <tt><span>dev-repo</span></tt> has been out-of-the-loop; changeset de61 is still <em>draft</em> there. If we’re not careful, we might mutate history in <tt><span>dev-repo</span></tt>, obsoleting a changeset that is already public. Let’s avoid that situation for now by pushing up to <tt><span>dev-repo</span></tt>:
</p>
<div>
<pre>$ hg push ../dev-repo
pushing to ../dev-repo
searching for changes
no changes found
</pre>
</div>
<p>
Even though no <em>changesets</em> were pushed, Mercurial still pushed obsolescence markers and phase changes to <tt><span>dev-repo</span></tt>.
</p>
<p>
A final note: since this fix is now <em>public</em>, it is immutable. It’s no longer possible to amend it:
</p>
<div>
<pre>$ hg amend -m 'fix bug 37'
abort: cannot amend public changesets
</pre>
</div>
<p>
This is, after all, the whole point of Mercurial’s phases: to prevent rewriting history that has already been published.
</p>
</div>
</div>
<div id="sharing-with-multiple-developers-code-review">
<h2>
<a href="#id10">Sharing with multiple developers: code review</a><a href="#sharing-with-multiple-developers-code-review" title="Permalink to this headline"></a>
</h2>
<p>
Now that you know how to share your own mutable history across multiple computers, you might be wondering if it makes sense to share mutable history with others. It does, but you have to be careful, stay alert, and <em>communicate</em> with your peers.
</p>
<p>
Code review is a good use case for sharing mutable history across multiple developers: Alice commits a draft changeset, submits it for review, and amends her changeset until her reviewer is satisfied. Meanwhile, Bob is also committing draft changesets for review, amending until his reviewer is satisfied. Once a particular changeset passes review, the respective author (Alice or Bob) pushes it to the public (publishing) repository.
</p>
<p>
Incidentally, the reviewers here can be anyone: maybe Bob and Alice review each other’s work; maybe the same third party reviews both; or maybe they pick different experts to review their work on different parts of a large codebase. Similarly, it doesn’t matter if reviews are conducted in person, by email, or by carrier pigeon. Code review is outside of the scope of Mercurial, so all we’re looking at here is the mechanics of committing, amending, pushing, and pulling.
</p>
<div id="id2">
<h3>
<a href="#id11">Setting up</a><a href="#id2" title="Permalink to this headline"></a>
</h3>
<p>
To demonstrate, let’s start with the <tt><span>public</span></tt> repository as we left it in the last example, with two immutable changesets (figure 5 above). We’ll clone a <tt><span>review</span></tt> repository from it, and then Alice and Bob will both clone from <tt><span>review</span></tt>.
</p>
<div>
<pre>$ hg clone public review
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review alice
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review bob
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
<p>
We need to configure Alice’s and Bob’s working repositories to enable <tt><span>evolve</span></tt>. First, edit Alice’s configuration with
</p>
<div>
<pre>$ hg -R alice config --edit --local
</pre>
</div>
<p>
and add
</p>
<p>
Then edit Bob’s repository configuration:
</p>
<div>
<pre>$ hg -R bob config --edit --local
</pre>
</div>
<p>
and add the same text.
</p>
</div>
<div id="example-3-alice-commits-and-amends-a-draft-fix">
<h3>
<a href="#id12">Example 3: Alice commits and amends a draft fix</a><a href="#example-3-alice-commits-and-amends-a-draft-fix" title="Permalink to this headline"></a>
</h3>
<p>
We’ll follow Alice working on a bug fix. We’re going to use bookmarks to make it easier to understand multiple branch heads in the <tt><span>review</span></tt> repository, so Alice starts off by creating a bookmark and committing her first attempt at a fix:
</p>
<div>
<pre>$ hg bookmark bug15
$ echo 'fix' &gt; file2
$ hg commit -A -u alice -m 'fix bug 15 (v1)'
adding file2
</pre>
</div>
<p>
Note the unorthodox “(v1)” in the commit message. We’re just using that to make this tutorial easier to follow; it’s not something we’d recommend in real life.
</p>
<p>
Of course Alice wouldn’t commit unless her fix worked to her satisfaction, so it must be time to solicit a code review. She does this by pushing to the <tt><span>review</span></tt> repository:
</p>
<div>
<pre>$ hg push -B bug15
[...]
added 1 changesets with 1 changes to 1 files
exporting bookmark bug15
</pre>
</div>
<p>
(The use of <tt><span>-B</span></tt> is important to ensure that we only push the bookmarked head, and that the bookmark itself is pushed. See this <a href="http://mercurial.aragost.com/kick-start/en/bookmarks/">guide to bookmarks</a>, especially the <a href="http://mercurial.aragost.com/kick-start/en/bookmarks/#sharing-bookmarks">Sharing Bookmarks</a> section, if you’re not familiar with bookmarks.)
</p>
<p>
Some time passes, and Alice receives her code review. As a result, Alice revises her fix and submits it for a second review:
</p>
<div>
<pre>$ echo 'Fix.' &gt; file2
$ hg amend -m 'fix bug 15 (v2)'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark bug15
</pre>
</div>
<p>
Figure 6 shows the state of the <tt><span>review</span></tt> repository at this point.
</p>
<blockquote>
<p>
[figure SG06: rev 2:fn1e is Alice’s obsolete v1, rev 3:cbdf is her v2; both children of rev 1:de61]
</p>
</blockquote>
<p>
After a busy morning of bug fixing, Alice stops for lunch. Let’s see what Bob has been up to.
</p>
</div>
<div id="example-4-bob-implements-and-publishes-a-new-feature">
<h3>
<a href="#id13">Example 4: Bob implements and publishes a new feature</a><a href="#example-4-bob-implements-and-publishes-a-new-feature" title="Permalink to this headline"></a>
</h3>
<p>
Meanwhile, Bob has been working on a new feature. Like Alice, he’ll use a bookmark to track his work, and he’ll push that bookmark to the <tt><span>review</span></tt> repository, so that reviewers know which changesets to review.
</p>
<div>
<pre>$ cd ../bob
$ echo 'stuff' &gt; file1
$ hg bookmark featureX
$ hg commit -u bob -m 'implement feature X (v1)' # rev 4:1636
$ hg push -B featureX
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
exporting bookmark featureX
</pre>
</div>
<p>
When Bob receives his code review, he improves his implementation a bit, amends, and submits the resulting changeset for review:
</p>
<div>
<pre>$ echo 'do stuff' &gt; file1
$ hg amend -m 'implement feature X (v2)' # rev 5:0eb7
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX
</pre>
</div>
<p>
Unfortunately, that still doesn’t pass muster. Bob’s reviewer insists on proper capitalization and punctuation.
</p>
<div>
<pre>$ echo 'Do stuff.' &gt; file1
$ hg amend -m 'implement feature X (v3)' # rev 6:540b
</pre>
</div>
<p>
On the bright side, the second review said, “Go ahead and publish once you fix that.” So Bob immediately publishes his third attempt:
</p>
<div>
<pre>$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
<p>
It’s not enough just to update <tt><span>public</span></tt>, though! Other people also use the <tt><span>review</span></tt> repository, and right now it doesn’t have Bob’s latest amendment (“v3”, revision 6:540b), nor does it know that the precursor of that changeset (“v2”, revision 5:0eb7) is obsolete. Thus, Bob pushes to <tt><span>review</span></tt> as well:
</p>
<div>
<pre>$ hg push ../review
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX
</pre>
</div>
<p>
Figure 7 shows the result of Bob’s work in both <tt><span>review</span></tt> and <tt><span>public</span></tt>.
</p>
<blockquote>
<p>
[figure SG07: review includes Alice’s draft work on bug 15, as well as Bob’s v1, v2, and v3 changes for feature X: v1 and v2 obsolete, v3 public. public contains only the final, public implementation of feature X]
</p>
</blockquote>
<p>
Incidentally, it’s important that Bob push to <tt><span>public</span></tt> <em>before</em> <tt><span>review</span></tt>. If he pushed to <tt><span>review</span></tt> first, then revision 6:540b would still be in <em>draft</em> phase in <tt><span>review</span></tt>, but it would be <em>public</em> in both Bob’s local repository and the <tt><span>public</span></tt> repository. That could lead to confusion at some point, which is easily avoided by pushing first to <tt><span>public</span></tt>.
</p>
</div>
<div id="example-5-alice-integrates-and-publishes">
<h3>
<a href="#id14">Example 5: Alice integrates and publishes</a><a href="#example-5-alice-integrates-and-publishes" title="Permalink to this headline"></a>
</h3>
<p>
Finally, Alice gets back from lunch and sees that the carrier pigeon with her second review has arrived (or maybe it’s in her email inbox). Alice’s reviewer approved her amended changeset, so she pushes it to <tt><span>public</span></tt>:
</p>
<div>
<pre>$ hg push ../public
[...]
remote has heads on branch 'default' that are not known locally: 540ba8f317e6
abort: push creates new remote head cbdfbd5a5db2!
(pull and merge or see "hg help push" for details about pushing new heads)
</pre>
</div>
<p>
Oops! Bob has won the race to push first to <tt><span>public</span></tt>. So Alice needs to integrate with Bob: let’s pull his changeset(s) and see what the branch heads are.
</p>
<div>
<pre>$ hg pull ../public
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
$ hg log -G -q -r 'head()' --template '{rev}:{node|short} ({author})\n'
o 5:540ba8f317e6 (bob)
|
| @ 4:cbdfbd5a5db2 (alice)
|/
</pre>
</div>
<p>
We’ll assume Alice and Bob are perfectly comfortable with rebasing changesets. (After all, they’re already using mutable history in the form of <tt><span>amend</span></tt>.) So Alice rebases her changeset on top of Bob’s and publishes the result:
</p>
<div>
<pre>$ hg rebase -d 5
$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
$ hg push ../review
[...]
added 1 changesets with 0 changes to 0 files
updating bookmark bug15
</pre>
</div>
<p>
The result, in both <tt><span>review</span></tt> and <tt><span>public</span></tt> repositories, is shown in figure 8.
</p>
<blockquote>
<p>
[figure SG08: review shows v1 and v2 of Alice’s fix, then v1, v2, v3 of Bob’s feature, finally Alice’s fix rebased onto Bob’s. public just shows the final public version of each changeset]
</p>
</blockquote>
</div>
</div>
<div id="getting-into-trouble-with-shared-mutable-history">
<h2>
<a href="#id15">Getting into trouble with shared mutable history</a><a href="#getting-into-trouble-with-shared-mutable-history" title="Permalink to this headline"></a>
</h2>
<p>
Mercurial with <tt><span>evolve</span></tt> is a powerful tool, and using powerful tools can have consequences. (You can cut yourself badly with a sharp knife, but every competent chef keeps several around. Ever try to chop onions with a spoon?)
</p>
<p>
In the user guide, we saw examples of <em>unstbale</em> changesets, which are the most common type of troubled changeset. (Recall that a non-obsolete changeset with obsolete ancestors is an orphan.)
</p>
<p>
Two other types of troubles can happen: <em>divergent</em> and <em>bumped</em> changesets. Both are more likely with shared mutable history, especially mutable history shared by multiple developers.
</p>
<div id="id3">
<h3>
<a href="#id16">Setting up</a><a href="#id3" title="Permalink to this headline"></a>
</h3>
<p>
For these examples, we’re going to use a slightly different workflow: as before, Alice and Bob share a <tt><span>public</span></tt> repository. But this time there is no <tt><span>review</span></tt> repository. Instead, Alice and Bob put on their cowboy hats, throw good practice to the wind, and pull directly from each other’s working repositories.
</p>
<p>
So we throw away everything except <tt><span>public</span></tt> and reclone:
</p>
<div>
<pre>$ rm -rf review alice bob
$ hg clone public alice
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone public bob
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
<p>
Once again we have to configure their repositories: enable <tt><span>evolve</span></tt> and (since Alice and Bob will be pulling directly from each other) make their repositories non-publishing. Edit Alice’s configuration:
</p>
<div>
<pre>$ hg -R alice config --edit --local
</pre>
</div>
<p>
and add
</p>
<div>
<pre>[extensions]
rebase =
evolve =
[phases]
publish = false
</pre>
</div>
<p>
Then edit Bob’s repository configuration:
</p>
<div>
<pre>$ hg -R bob config --edit --local
</pre>
</div>
<p>
and add the same text.
</p>
</div>
<div id="example-6-divergent-changesets">
<h3>
<a href="#id17">Example 6: Divergent changesets</a><a href="#example-6-divergent-changesets" title="Permalink to this headline"></a>
</h3>
<p>
When an obsolete changeset has two successors, those successors are <em>divergent</em>. One way to get into such a situation is by failing to communicate with your teammates. Let’s see how that might happen.
</p>
<p>
First, we’ll have Bob commit a bug fix that could still be improved:
</p>
<div>
<pre>$ cd bob
$ echo 'pretty good fix' &gt;&gt; file1
$ hg commit -u bob -m 'fix bug 24 (v1)' # rev 4:2fe6
</pre>
</div>
<p>
Since Alice and Bob are now in cowboy mode, Alice pulls Bob’s draft changeset and amends it herself.
</p>
<div>
<pre>$ cd ../alice
$ hg pull -u ../bob
[...]
added 1 changesets with 1 changes to 1 files
$ echo 'better fix (alice)' &gt;&gt; file1
$ hg amend -u alice -m 'fix bug 24 (v2 by alice)'
</pre>
</div>
<p>
But Bob has no idea that Alice just did this. (See how important good communication is?) So he implements a better fix of his own:
</p>
<div>
<pre>$ cd ../bob
$ echo 'better fix (bob)' &gt;&gt; file1
$ hg amend -u bob -m 'fix bug 24 (v2 by bob)' # rev 6:a360
</pre>
</div>
<p>
At this point, the divergence exists, but only in theory: Bob’s original changeset, 4:2fe6, is obsolete and has two successors. But those successors are in different repositories, so the trouble is not visible to anyone yet. It will be as soon as Bob pulls from Alice’s repository (or vice-versa).
</p>
<div>
<pre>$ hg pull ../alice
[...]
added 1 changesets with 1 changes to 2 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
2 new divergent changesets
</pre>
</div>
<p>
Figure 9 shows the situation in Bob’s repository.
</p>
<blockquote>
<p>
[figure SG09: Bob’s repo with 2 heads for the 2 divergent changesets, 6:a360 and 7:e3f9; wc is at 6:a360; both are successors of obsolete 4:2fe6, hence divergence]
</p>
</blockquote>
<p>
Now we need to get out of trouble. As usual, the answer is to evolve history.
</p>
<div>
<pre>$ HGMERGE=internal:other hg evolve
merge:[6] fix bug 24 (v2 by bob)
with: [7] fix bug 24 (v2 by alice)
base: [4] fix bug 24 (v1)
0 files updated, 1 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
<p>
Figure 10 shows how Bob’s repository looks now.
</p>
<blockquote>
<p>
[figure SG10: only one visible head, 9:5ad6, successor to hidden 6:a360 and 7:e3f9]
</p>
</blockquote>
<p>
We carefully dodged a merge conflict by specifying a merge tool (<tt><span>internal:other</span></tt>) that will take Alice’s changes over Bob’s. (You might wonder why Bob wouldn’t prefer his own changes by using <tt><span>internal:local</span></tt>. He’s avoiding a <a href="#bug">bug</a> in <tt><span>evolve</span></tt> that occurs when evolving divergent changesets using <tt><span>internal:local</span></tt>.)
</p>
<p>
# XXX this link does not work .. <span id="bug">bug</span>: <a href="https://bitbucket.org/marmoute/mutable-history/issue/48/">https://bitbucket.org/marmoute/mutable-history/issue/48/</a>
</p>
<p>
** STOP HERE: WORK IN PROGRESS **
</p>
</div>
<div id="phase-divergence-when-a-rewritten-changeset-is-made-public">
<h3>
<a href="#id18">Phase-divergence: when a rewritten changeset is made public</a><a href="#phase-divergence-when-a-rewritten-changeset-is-made-public" title="Permalink to this headline"></a>
</h3>
<p>
If Alice and Bob are collaborating on some mutable changesets, it’s possible to get into a situation where an otherwise worthwhile changeset cannot be pushed to the public repository; it is <em>phase-divergent</em> with another changeset that was made public first. Let’s demonstrate one way this could happen.
</p>
<p>
It starts with Alice committing a bug fix. Right now, we don’t yet know if this bug fix is good enough to push to the public repository, but it’s good enough for Alice to commit.
</p>
<div>
<pre>$ cd alice
$ echo 'fix' &gt; file2
$ hg commit -A -m 'fix bug 15'
adding file2
</pre>
</div>
<p>
Now Bob has a bad idea: he decides to pull whatever Alice is working on and tweak her bug fix to his taste:
</p>
<div>
<pre>$ cd ../bob
$ hg pull -u ../alice
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ echo 'Fix.' &gt; file2
$ hg amend -A -m 'fix bug 15 (amended)'
</pre>
</div>
<p>
(Note the lack of communication between Alice and Bob. Failing to communicate with your colleagues is a good way to get into trouble. Nevertheless, <tt><span>evolve</span></tt> can usually sort things out, as we will see.)
</p>
<blockquote>
<p>
[figure SG06: Bob’s repo with one amendment]
</p>
</blockquote>
<p>
After some testing, Alice realizes her bug fix is just fine as it is: no need for further polishing and amending, this changeset is ready to publish.
</p>
<div>
<pre>$ cd ../alice
$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
<p>
This introduces a contradiction: in Bob’s repository, changeset 2:e011 (his copy of Alice’s fix) is obsolete, since Bob amended it. But in Alice’s repository (and the <tt><span>public</span></tt> repository), that changeset is public: it is immutable, carved in stone for all eternity. No changeset can be both obsolete and public, so Bob is in for a surprise the next time he pulls from <tt><span>public</span></tt>:
</p>
<div>
<pre>$ cd ../bob
$ hg pull -q -u
1 new phase-divergent changesets
</pre>
</div>
<p>
Figure 7 shows what just happened to Bob’s repository: changeset 2:e011 is now public, so it can’t be obsolete. When that changeset was obsolete, it made perfect sense for it to have a successor, namely Bob’s amendment of Alice’s fix (changeset 4:fe88). But it’s illogical for a public changeset to have a successor, so 4:fe88 is troubled: it has become <em>bumped</em>.
</p>
<blockquote>
<p>
[figure SG07: 2:e011 now public not obsolete, 4:fe88 now bumped]
</p>
</blockquote>
<p>
As usual when there’s trouble in your repository, the solution is to evolve it:
</p>
<p>
Figure 8 illustrates Bob’s repository after evolving away the bumped changeset. Ignoring the obsolete changesets, Bob now has a nice, clean, simple history. His amendment of Alice’s bug fix lives on, as changeset 5:227d—albeit with a software-generated commit message. (Bob should probably amend that changeset to improve the commit message.) But the important thing is that his repository no longer has any troubled changesets, thanks to <tt><span>evolve</span></tt>.
</p>
<blockquote>
<p>
[figure SG08: 5:227d is new, formerly bumped changeset 4:fe88 now hidden]
</p>
</blockquote>
</div>
</div>
<div id="conclusion">
<h2>
<a href="#id19">Conclusion</a><a href="#conclusion" title="Permalink to this headline"></a>
</h2>
<p>
Mutable history is a powerful tool. Like a sharp knife, an experienced user can do wonderful things with it, much more wonderful than with a dull knife (never mind a rusty spoon). At the same time, an inattentive or careless user can do harm to himself or others. Mercurial with <tt><span>evolve</span></tt> goes to great lengths to limit the harm you can do by trying to handle all possible types of “troubled” changesets. Nevertheless, having a first-aid kit nearby does not mean you should stop being careful with sharp knives.
</p>
<p>
Mutable history shared across multiple repositories by a single developer is a natural extension of this model. Once you are used to using a single sharp knife on its own, it’s pretty straightforward to chop onions and mushrooms using the same knife, or to alternate between two chopping boards with different knives.
</p>
<p>
Mutable history shared by multiple developers is a scary place to go. Imagine a professional kitchen full of expert chefs tossing their favourite knives back and forth, with the occasional axe or chainsaw thrown in to spice things up. If you’re confident that you <em>and your colleagues</em> can do it without losing a limb, go for it. But be sure to practice a lot first before you rely on it!
</p>
</div>
</div>

979
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/mercurial/source.html vendored Normal file
View File

@ -0,0 +1,979 @@
<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>
Evolve: Shared Mutable History — evolve extension for Mercurial
</title>
<link rel="stylesheet" href="_static/haiku.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<script type="text/javascript">
//<![CDATA[
var DOCUMENTATION_OPTIONS = {
URL_ROOT: './',
VERSION: '0.0',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
//]]>
</script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<link rel="shortcut icon" href="_static/logo-evolve.ico" />
<link rel="top" title="evolve extension for Mercurial" href="index.html" />
<link rel="next" title="Evolve: Concepts" href="concepts.html" />
<link rel="prev" title="Evolve: User Guide" href="user-guide.html" />
</head>
<body>
<div class="header">
<h1 class="heading">
<a href="index.html"><span>evolve extension for Mercurial</span></a>
</h1>
<h2 class="heading">
<span>Evolve: Shared Mutable History</span>
</h2>
</div>
<div class="topnav">
<p>
«&#160;&#160;<a href="user-guide.html">Evolve: User Guide</a> &#160;&#160;::&#160;&#160; <a class="uplink" href="index.html">Contents</a> &#160;&#160;::&#160;&#160; <a href="concepts.html">Evolve: Concepts</a>&#160;&#160;»
</p>
</div>
<div class="content">
<div class="section" id="evolve-shared-mutable-history">
<h1>
<a class="toc-backref" href="#id4">Evolve: Shared Mutable History</a><a class="headerlink" href="#evolve-shared-mutable-history" title="Permalink to this headline"></a>
</h1>
<div class="contents topic" id="contents">
<p class="topic-title first">
Contents
</p>
<ul class="simple">
<li>
<a class="reference internal" href="#evolve-shared-mutable-history" id="id4">Evolve: Shared Mutable History</a>
<ul>
<li>
<a class="reference internal" href="#sharing-with-a-single-developer" id="id5">Sharing with a single developer</a>
<ul>
<li>
<a class="reference internal" href="#publishing-and-non-publishing-repositories" id="id6">Publishing and non-publishing repositories</a>
</li>
<li>
<a class="reference internal" href="#setting-up" id="id7">Setting up</a>
</li>
<li>
<a class="reference internal" href="#example-1-amend-a-shared-changeset" id="id8">Example 1: Amend a shared changeset</a>
</li>
<li>
<a class="reference internal" href="#example-2-amend-again-locally" id="id9">Example 2: Amend again, locally</a>
</li>
</ul>
</li>
<li>
<a class="reference internal" href="#sharing-with-multiple-developers-code-review" id="id10">Sharing with multiple developers: code review</a>
<ul>
<li>
<a class="reference internal" href="#id2" id="id11">Setting up</a>
</li>
<li>
<a class="reference internal" href="#example-3-alice-commits-and-amends-a-draft-fix" id="id12">Example 3: Alice commits and amends a draft fix</a>
</li>
<li>
<a class="reference internal" href="#example-4-bob-implements-and-publishes-a-new-feature" id="id13">Example 4: Bob implements and publishes a new feature</a>
</li>
<li>
<a class="reference internal" href="#example-5-alice-integrates-and-publishes" id="id14">Example 5: Alice integrates and publishes</a>
</li>
</ul>
</li>
<li>
<a class="reference internal" href="#getting-into-trouble-with-shared-mutable-history" id="id15">Getting into trouble with shared mutable history</a>
<ul>
<li>
<a class="reference internal" href="#id3" id="id16">Setting up</a>
</li>
<li>
<a class="reference internal" href="#example-6-divergent-changesets" id="id17">Example 6: Divergent changesets</a>
</li>
<li>
<a class="reference internal" href="#phase-divergence-when-a-rewritten-changeset-is-made-public" id="id18">Phase-divergence: when a rewritten changeset is made public</a>
</li>
</ul>
</li>
<li>
<a class="reference internal" href="#conclusion" id="id19">Conclusion</a>
</li>
</ul>
</li>
</ul>
</div>
<p>
Once you have mastered the art of mutable history in a single repository (see the <a class="reference external" href="user-guide.html">user guide</a>), you can move up to the next level: <em>shared</em> mutable history. <tt class="docutils literal"><span class="pre">evolve</span></tt> lets you push and pull draft changesets between repositories along with their obsolescence markers. This opens up a number of interesting possibilities.
</p>
<p>
The simplest scenario is a single developer working across two computers. Say you’re working on code that must be tested on a remote test server, probably in a rack somewhere, only accessible by SSH, and running an “enterprise-grade” (out-of-date) OS. But you probably prefer to write code locally: everything is setup the way you like it, and you can use your preferred editor, IDE, merge/diff tools, etc.
</p>
<p>
Traditionally, your options are limited: either
</p>
<blockquote>
<div>
<ul class="simple">
<li>(ab)use your source control system by committing half-working code in order to get it onto the remote test server, or
</li>
<li>go behind source control’s back by using <tt class="docutils literal"><span class="pre">rsync</span></tt> (or similar) to transfer your code back-and-forth until it is ready to commit
</li>
</ul>
</div>
</blockquote>
<p>
The former is less bad with distributed version control systems like Mercurial, but it’s still far from ideal. (One important version control “best practice” is that every commit should make things just a little bit better, i.e. you should never commit code that is worse than what came before.) The latter, avoiding version control entirely, means that you’re walking a tightrope without a safety net. One accidental <tt class="docutils literal"><span class="pre">rsync</span></tt> in the wrong direction could destroy hours of work.
</p>
<p>
Using Mercurial with <tt class="docutils literal"><span class="pre">evolve</span></tt> to share mutable history solves these problems. As with single-repository <tt class="docutils literal"><span class="pre">evolve</span></tt>, you can commit whenever the code is demonstrably better, even if all the tests aren’t passing yet—just <tt class="docutils literal"><span class="pre">hg</span> <span class="pre">amend</span></tt> when they are. And you can transfer those half-baked changesets between repositories to try things out on your test server before anything is carved in stone.
</p>
<p>
A less common scenario is multiple developers sharing mutable history, typically for code review. We’ll cover this scenario later. First, we will cover single-user sharing.
</p>
<div class="section" id="sharing-with-a-single-developer">
<h2>
<a class="toc-backref" href="#id5">Sharing with a single developer</a><a class="headerlink" href="#sharing-with-a-single-developer" title="Permalink to this headline"></a>
</h2>
<div class="section" id="publishing-and-non-publishing-repositories">
<h3>
<a class="toc-backref" href="#id6">Publishing and non-publishing repositories</a><a class="headerlink" href="#publishing-and-non-publishing-repositories" title="Permalink to this headline"></a>
</h3>
<p>
The key to shared mutable history is to keep your changesets in <em>draft</em> phase as you pass them around. Recall that by default, <tt class="docutils literal"><span class="pre">hg</span> <span class="pre">push</span></tt> promotes changesets from <em>draft</em> to <em>public</em>, and public changesets are immutable. You can change this behaviour by reconfiguring the <em>remote</em> repository so that it is non-publishing. (Short version: set <tt class="docutils literal"><span class="pre">phases.publish</span></tt> to <tt class="docutils literal"><span class="pre">false</span></tt>. Long version follows.)
</p>
</div>
<div class="section" id="setting-up">
<h3>
<a class="toc-backref" href="#id7">Setting up</a><a class="headerlink" href="#setting-up" title="Permalink to this headline"></a>
</h3>
<p>
We’ll work through an example with three local repositories, although in the real world they’d most likely be on three different computers. First, the <tt class="docutils literal"><span class="pre">public</span></tt> repository is where tested, polished changesets live, and it is where you synchronize with the rest of your team.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg init public
</pre>
</div>
</div>
<p>
We’ll need two clones where work gets done, <tt class="docutils literal"><span class="pre">test-repo</span></tt> and <tt class="docutils literal"><span class="pre">dev-repo</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg clone public test-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone test-repo dev-repo
updating to branch default
0 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
<tt class="docutils literal"><span class="pre">dev-repo</span></tt> is your local machine, with GUI merge tools and IDEs and everything configured just the way you like it. <tt class="docutils literal"><span class="pre">test-repo</span></tt> is the test server in a rack somewhere behind SSH. So for the most part, we’ll develop in <tt class="docutils literal"><span class="pre">dev-repo</span></tt>, push to <tt class="docutils literal"><span class="pre">test-repo</span></tt>, test and polish there, and push to <tt class="docutils literal"><span class="pre">public</span></tt>.
</p>
<p>
The key to shared mutable history is to make the target repository, in this case <tt class="docutils literal"><span class="pre">test-repo</span></tt>, non-publishing. And, of course, we have to enable the <tt class="docutils literal"><span class="pre">evolve</span></tt> extension in both <tt class="docutils literal"><span class="pre">test-repo</span></tt> and <tt class="docutils literal"><span class="pre">dev-repo</span></tt>.
</p>
<p>
First, edit the configuration for <tt class="docutils literal"><span class="pre">test-repo</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R test-repo config --edit --local
</pre>
</div>
</div>
<p>
and add
</p>
<div class="highlight-python">
<div class="highlight">
<pre>[phases]
publish = false
[extensions]
evolve =
</pre>
</div>
</div>
<p>
Then edit the configuration for <tt class="docutils literal"><span class="pre">dev-repo</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R dev-repo config --edit --local
</pre>
</div>
</div>
<p>
and add
</p>
<div class="highlight-python">
<div class="highlight">
<pre>[extensions]
evolve =
</pre>
</div>
</div>
<p>
Keep in mind that in real life, these repositories would probably be on separate computers, so you’d have to login to each one to configure each repository.
</p>
<p>
To start things off, let’s make one public, immutable changeset:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd test-repo
$ echo 'my new project' &gt; file1
$ hg add file1
$ hg commit -m 'create new project'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
</div>
<p>
and pull that into the development repository:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
</div>
<div class="section" id="example-1-amend-a-shared-changeset">
<h3>
<a class="toc-backref" href="#id8">Example 1: Amend a shared changeset</a><a class="headerlink" href="#example-1-amend-a-shared-changeset" title="Permalink to this headline"></a>
</h3>
<p>
Everything you learned in the <a class="reference external" href="user-guide.html">user guide</a> applies to work done in <tt class="docutils literal"><span class="pre">dev-repo</span></tt>. You can commit, amend, uncommit, evolve, and so forth just as before.
</p>
<p>
Things get different when you push changesets to <tt class="docutils literal"><span class="pre">test-repo</span></tt>. Or rather, things stay the same, which <em>is</em> different: because we configured <tt class="docutils literal"><span class="pre">test-repo</span></tt> to be non-publishing, draft changesets stay draft when we push them to <tt class="docutils literal"><span class="pre">test-repo</span></tt>. Importantly, they’re also draft (mutable) in <tt class="docutils literal"><span class="pre">test-repo</span></tt>.
</p>
<p>
Let’s commit a preliminary change and push it to <tt class="docutils literal"><span class="pre">test-repo</span></tt> for testing.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'fix fix fix' &gt; file1
$ hg commit -m 'prelim change'
$ hg push ../test-repo
</pre>
</div>
</div>
<p>
At this point, <tt class="docutils literal"><span class="pre">dev-repo</span></tt> and <tt class="docutils literal"><span class="pre">test-repo</span></tt> have the same changesets in the same phases:
</p>
<blockquote>
<div>
[figure SG01: rev 0:0dc9 public, rev 1:f649 draft, same on both repos]
</div>
</blockquote>
<p>
(You may notice a change in notation from the user guide: now changesets are labelled with their revision number and the first four digits of the 40-digit hexadecimal changeset ID. Mercurial revision numbers are never stable when working across repositories, especially when obsolescence is involved. We’ll see why shortly.)
</p>
<p>
Now let’s switch to <tt class="docutils literal"><span class="pre">test-repo</span></tt> to test our change:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../test-repo
$ hg update
</pre>
</div>
</div>
<p>
Don’t forget to <tt class="docutils literal"><span class="pre">hg</span> <span class="pre">update</span></tt>! Pushing only adds changesets to a remote repository; it does not update the working directory (unless you have a hook that updates for you).
</p>
<p>
Now let’s imagine the tests failed because we didn’t use proper punctuation and capitalization (oops). Let’s amend our preliminary fix (and fix the lame commit message while we’re at it):
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'Fix fix fix.' &gt; file1
$ hg amend -m 'fix bug 37'
</pre>
</div>
</div>
<p>
Now we’re in a funny intermediate state (figure 2): revision 1:f649 is obsolete in <tt class="docutils literal"><span class="pre">test-repo</span></tt>, having been replaced by revision 3:60ff (revision 2:2a03 is another one of those temporary amend commits that we saw in the user guide)—but <tt class="docutils literal"><span class="pre">dev-repo</span></tt> knows nothing of these recent developments.
</p>
<blockquote>
<div>
[figure SG02: test-repo has rev 0:0dc9 public, rev 1:f649, 2:2a03 obsolete, rev 3:60ff draft; dev-repo same as in SG01]
</div>
</blockquote>
<p>
Let’s resynchronize:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../dev-repo
$ hg pull -u
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
As seen in figure 3, this transfers the new changeset <em>and</em> the obsolescence marker for revision 1. However, it does <em>not</em> transfer the temporary amend commit, because it is hidden. Push and pull transfer obsolescence markers between repositories, but they do not transfer hidden changesets.
</p>
<blockquote>
<div>
[figure SG03: dev-repo grows new rev 2:60ff, marks 1:f649 obsolete]
</div>
</blockquote>
<p>
Because of this deliberately incomplete synchronization, revision numbers in <tt class="docutils literal"><span class="pre">test-repo</span></tt> and <tt class="docutils literal"><span class="pre">dev-repo</span></tt> are no longer consistent. We <em>must</em> use changeset IDs.
</p>
</div>
<div class="section" id="example-2-amend-again-locally">
<h3>
<a class="toc-backref" href="#id9">Example 2: Amend again, locally</a><a class="headerlink" href="#example-2-amend-again-locally" title="Permalink to this headline"></a>
</h3>
<p>
This process can repeat. Perhaps you figure out a more elegant fix to the bug, and want to mutate history so nobody ever knows you had a less-than-perfect idea. We’ll implement it locally in <tt class="docutils literal"><span class="pre">dev-repo</span></tt> and push to <tt class="docutils literal"><span class="pre">test-repo</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'Fix, fix, and fix.' &gt; file1
$ hg amend
$ hg push
</pre>
</div>
</div>
<p>
This time around, the temporary amend commit is in <tt class="docutils literal"><span class="pre">dev-repo</span></tt>, and it is not transferred to <tt class="docutils literal"><span class="pre">test-repo</span></tt>—the same as before, just in the opposite direction. Figure 4 shows the two repositories after amending in <tt class="docutils literal"><span class="pre">dev-repo</span></tt> and pushing to <tt class="docutils literal"><span class="pre">test-repo</span></tt>.
</p>
<blockquote>
<div>
[figure SG04: each repo has one temporary amend commit, but they’re different in each one]
</div>
</blockquote>
<p>
Let’s hop over to <tt class="docutils literal"><span class="pre">test-repo</span></tt> to test the more elegant fix:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../test-repo
$ hg update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
This time, all the tests pass, so no further amending is required. This bug fix is finished, so we push it to the public repository:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
</div>
<p>
Note that only one changeset—the final version, after two amendments—was actually pushed. Again, Mercurial doesn’t transfer hidden changesets on push and pull.
</p>
<p>
So the picture in <tt class="docutils literal"><span class="pre">public</span></tt> is much simpler than in either <tt class="docutils literal"><span class="pre">dev-repo</span></tt> or <tt class="docutils literal"><span class="pre">test-repo</span></tt>. Neither of our missteps nor our amendments are publicly visible, just the final, beautifully polished changeset:
</p>
<blockquote>
<div>
[figure SG05: public repo with rev 0:0dc9, 1:de61, both public]
</div>
</blockquote>
<p>
There is one important step left to do. Because we pushed from <tt class="docutils literal"><span class="pre">test-repo</span></tt> to <tt class="docutils literal"><span class="pre">public</span></tt>, the pushed changeset is in <em>public</em> phase in those two repositories. But <tt class="docutils literal"><span class="pre">dev-repo</span></tt> has been out-of-the-loop; changeset de61 is still <em>draft</em> there. If we’re not careful, we might mutate history in <tt class="docutils literal"><span class="pre">dev-repo</span></tt>, obsoleting a changeset that is already public. Let’s avoid that situation for now by pushing up to <tt class="docutils literal"><span class="pre">dev-repo</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push ../dev-repo
pushing to ../dev-repo
searching for changes
no changes found
</pre>
</div>
</div>
<p>
Even though no <em>changesets</em> were pushed, Mercurial still pushed obsolescence markers and phase changes to <tt class="docutils literal"><span class="pre">dev-repo</span></tt>.
</p>
<p>
A final note: since this fix is now <em>public</em>, it is immutable. It’s no longer possible to amend it:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg amend -m 'fix bug 37'
abort: cannot amend public changesets
</pre>
</div>
</div>
<p>
This is, after all, the whole point of Mercurial’s phases: to prevent rewriting history that has already been published.
</p>
</div>
</div>
<div class="section" id="sharing-with-multiple-developers-code-review">
<h2>
<a class="toc-backref" href="#id10">Sharing with multiple developers: code review</a><a class="headerlink" href="#sharing-with-multiple-developers-code-review" title="Permalink to this headline"></a>
</h2>
<p>
Now that you know how to share your own mutable history across multiple computers, you might be wondering if it makes sense to share mutable history with others. It does, but you have to be careful, stay alert, and <em>communicate</em> with your peers.
</p>
<p>
Code review is a good use case for sharing mutable history across multiple developers: Alice commits a draft changeset, submits it for review, and amends her changeset until her reviewer is satisfied. Meanwhile, Bob is also committing draft changesets for review, amending until his reviewer is satisfied. Once a particular changeset passes review, the respective author (Alice or Bob) pushes it to the public (publishing) repository.
</p>
<p>
Incidentally, the reviewers here can be anyone: maybe Bob and Alice review each other’s work; maybe the same third party reviews both; or maybe they pick different experts to review their work on different parts of a large codebase. Similarly, it doesn’t matter if reviews are conducted in person, by email, or by carrier pigeon. Code review is outside of the scope of Mercurial, so all we’re looking at here is the mechanics of committing, amending, pushing, and pulling.
</p>
<div class="section" id="id2">
<h3>
<a class="toc-backref" href="#id11">Setting up</a><a class="headerlink" href="#id2" title="Permalink to this headline"></a>
</h3>
<p>
To demonstrate, let’s start with the <tt class="docutils literal"><span class="pre">public</span></tt> repository as we left it in the last example, with two immutable changesets (figure 5 above). We’ll clone a <tt class="docutils literal"><span class="pre">review</span></tt> repository from it, and then Alice and Bob will both clone from <tt class="docutils literal"><span class="pre">review</span></tt>.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg clone public review
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review alice
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone review bob
updating to branch default
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
We need to configure Alice’s and Bob’s working repositories to enable <tt class="docutils literal"><span class="pre">evolve</span></tt>. First, edit Alice’s configuration with
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R alice config --edit --local
</pre>
</div>
</div>
<p>
and add
</p>
<div class="highlight-python">
<div class="highlight">
<pre>[extensions]
evolve =
</pre>
</div>
</div>
<p>
Then edit Bob’s repository configuration:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R bob config --edit --local
</pre>
</div>
</div>
<p>
and add the same text.
</p>
</div>
<div class="section" id="example-3-alice-commits-and-amends-a-draft-fix">
<h3>
<a class="toc-backref" href="#id12">Example 3: Alice commits and amends a draft fix</a><a class="headerlink" href="#example-3-alice-commits-and-amends-a-draft-fix" title="Permalink to this headline"></a>
</h3>
<p>
We’ll follow Alice working on a bug fix. We’re going to use bookmarks to make it easier to understand multiple branch heads in the <tt class="docutils literal"><span class="pre">review</span></tt> repository, so Alice starts off by creating a bookmark and committing her first attempt at a fix:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg bookmark bug15
$ echo 'fix' &gt; file2
$ hg commit -A -u alice -m 'fix bug 15 (v1)'
adding file2
</pre>
</div>
</div>
<p>
Note the unorthodox “(v1)” in the commit message. We’re just using that to make this tutorial easier to follow; it’s not something we’d recommend in real life.
</p>
<p>
Of course Alice wouldn’t commit unless her fix worked to her satisfaction, so it must be time to solicit a code review. She does this by pushing to the <tt class="docutils literal"><span class="pre">review</span></tt> repository:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push -B bug15
[...]
added 1 changesets with 1 changes to 1 files
exporting bookmark bug15
</pre>
</div>
</div>
<p>
(The use of <tt class="docutils literal"><span class="pre">-B</span></tt> is important to ensure that we only push the bookmarked head, and that the bookmark itself is pushed. See this <a class="reference external" href="http://mercurial.aragost.com/kick-start/en/bookmarks/">guide to bookmarks</a>, especially the <a class="reference external" href="http://mercurial.aragost.com/kick-start/en/bookmarks/#sharing-bookmarks">Sharing Bookmarks</a> section, if you’re not familiar with bookmarks.)
</p>
<p>
Some time passes, and Alice receives her code review. As a result, Alice revises her fix and submits it for a second review:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'Fix.' &gt; file2
$ hg amend -m 'fix bug 15 (v2)'
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark bug15
</pre>
</div>
</div>
<p>
Figure 6 shows the state of the <tt class="docutils literal"><span class="pre">review</span></tt> repository at this point.
</p>
<blockquote>
<div>
[figure SG06: rev 2:fn1e is Alice’s obsolete v1, rev 3:cbdf is her v2; both children of rev 1:de61]
</div>
</blockquote>
<p>
After a busy morning of bug fixing, Alice stops for lunch. Let’s see what Bob has been up to.
</p>
</div>
<div class="section" id="example-4-bob-implements-and-publishes-a-new-feature">
<h3>
<a class="toc-backref" href="#id13">Example 4: Bob implements and publishes a new feature</a><a class="headerlink" href="#example-4-bob-implements-and-publishes-a-new-feature" title="Permalink to this headline"></a>
</h3>
<p>
Meanwhile, Bob has been working on a new feature. Like Alice, he’ll use a bookmark to track his work, and he’ll push that bookmark to the <tt class="docutils literal"><span class="pre">review</span></tt> repository, so that reviewers know which changesets to review.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../bob
$ echo 'stuff' &gt; file1
$ hg bookmark featureX
$ hg commit -u bob -m 'implement feature X (v1)' # rev 4:1636
$ hg push -B featureX
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
exporting bookmark featureX
</pre>
</div>
</div>
<p>
When Bob receives his code review, he improves his implementation a bit, amends, and submits the resulting changeset for review:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'do stuff' &gt; file1
$ hg amend -m 'implement feature X (v2)' # rev 5:0eb7
$ hg push
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX
</pre>
</div>
</div>
<p>
Unfortunately, that still doesn’t pass muster. Bob’s reviewer insists on proper capitalization and punctuation.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ echo 'Do stuff.' &gt; file1
$ hg amend -m 'implement feature X (v3)' # rev 6:540b
</pre>
</div>
</div>
<p>
On the bright side, the second review said, “Go ahead and publish once you fix that.” So Bob immediately publishes his third attempt:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
</div>
<p>
It’s not enough just to update <tt class="docutils literal"><span class="pre">public</span></tt>, though! Other people also use the <tt class="docutils literal"><span class="pre">review</span></tt> repository, and right now it doesn’t have Bob’s latest amendment (“v3”, revision 6:540b), nor does it know that the precursor of that changeset (“v2”, revision 5:0eb7) is obsolete. Thus, Bob pushes to <tt class="docutils literal"><span class="pre">review</span></tt> as well:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push ../review
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
updating bookmark featureX
</pre>
</div>
</div>
<p>
Figure 7 shows the result of Bob’s work in both <tt class="docutils literal"><span class="pre">review</span></tt> and <tt class="docutils literal"><span class="pre">public</span></tt>.
</p>
<blockquote>
<div>
[figure SG07: review includes Alice’s draft work on bug 15, as well as Bob’s v1, v2, and v3 changes for feature X: v1 and v2 obsolete, v3 public. public contains only the final, public implementation of feature X]
</div>
</blockquote>
<p>
Incidentally, it’s important that Bob push to <tt class="docutils literal"><span class="pre">public</span></tt> <em>before</em> <tt class="docutils literal"><span class="pre">review</span></tt>. If he pushed to <tt class="docutils literal"><span class="pre">review</span></tt> first, then revision 6:540b would still be in <em>draft</em> phase in <tt class="docutils literal"><span class="pre">review</span></tt>, but it would be <em>public</em> in both Bob’s local repository and the <tt class="docutils literal"><span class="pre">public</span></tt> repository. That could lead to confusion at some point, which is easily avoided by pushing first to <tt class="docutils literal"><span class="pre">public</span></tt>.
</p>
</div>
<div class="section" id="example-5-alice-integrates-and-publishes">
<h3>
<a class="toc-backref" href="#id14">Example 5: Alice integrates and publishes</a><a class="headerlink" href="#example-5-alice-integrates-and-publishes" title="Permalink to this headline"></a>
</h3>
<p>
Finally, Alice gets back from lunch and sees that the carrier pigeon with her second review has arrived (or maybe it’s in her email inbox). Alice’s reviewer approved her amended changeset, so she pushes it to <tt class="docutils literal"><span class="pre">public</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg push ../public
[...]
remote has heads on branch 'default' that are not known locally: 540ba8f317e6
abort: push creates new remote head cbdfbd5a5db2!
(pull and merge or see "hg help push" for details about pushing new heads)
</pre>
</div>
</div>
<p>
Oops! Bob has won the race to push first to <tt class="docutils literal"><span class="pre">public</span></tt>. So Alice needs to integrate with Bob: let’s pull his changeset(s) and see what the branch heads are.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg pull ../public
[...]
added 1 changesets with 1 changes to 1 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
$ hg log -G -q -r 'head()' --template '{rev}:{node|short} ({author})\n'
o 5:540ba8f317e6 (bob)
|
| @ 4:cbdfbd5a5db2 (alice)
|/
</pre>
</div>
</div>
<p>
We’ll assume Alice and Bob are perfectly comfortable with rebasing changesets. (After all, they’re already using mutable history in the form of <tt class="docutils literal"><span class="pre">amend</span></tt>.) So Alice rebases her changeset on top of Bob’s and publishes the result:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg rebase -d 5
$ hg push ../public
[...]
added 1 changesets with 1 changes to 1 files
$ hg push ../review
[...]
added 1 changesets with 0 changes to 0 files
updating bookmark bug15
</pre>
</div>
</div>
<p>
The result, in both <tt class="docutils literal"><span class="pre">review</span></tt> and <tt class="docutils literal"><span class="pre">public</span></tt> repositories, is shown in figure 8.
</p>
<blockquote>
<div>
[figure SG08: review shows v1 and v2 of Alice’s fix, then v1, v2, v3 of Bob’s feature, finally Alice’s fix rebased onto Bob’s. public just shows the final public version of each changeset]
</div>
</blockquote>
</div>
</div>
<div class="section" id="getting-into-trouble-with-shared-mutable-history">
<h2>
<a class="toc-backref" href="#id15">Getting into trouble with shared mutable history</a><a class="headerlink" href="#getting-into-trouble-with-shared-mutable-history" title="Permalink to this headline"></a>
</h2>
<p>
Mercurial with <tt class="docutils literal"><span class="pre">evolve</span></tt> is a powerful tool, and using powerful tools can have consequences. (You can cut yourself badly with a sharp knife, but every competent chef keeps several around. Ever try to chop onions with a spoon?)
</p>
<p>
In the user guide, we saw examples of <em>unstbale</em> changesets, which are the most common type of troubled changeset. (Recall that a non-obsolete changeset with obsolete ancestors is an orphan.)
</p>
<p>
Two other types of troubles can happen: <em>divergent</em> and <em>bumped</em> changesets. Both are more likely with shared mutable history, especially mutable history shared by multiple developers.
</p>
<div class="section" id="id3">
<h3>
<a class="toc-backref" href="#id16">Setting up</a><a class="headerlink" href="#id3" title="Permalink to this headline"></a>
</h3>
<p>
For these examples, we’re going to use a slightly different workflow: as before, Alice and Bob share a <tt class="docutils literal"><span class="pre">public</span></tt> repository. But this time there is no <tt class="docutils literal"><span class="pre">review</span></tt> repository. Instead, Alice and Bob put on their cowboy hats, throw good practice to the wind, and pull directly from each other’s working repositories.
</p>
<p>
So we throw away everything except <tt class="docutils literal"><span class="pre">public</span></tt> and reclone:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ rm -rf review alice bob
$ hg clone public alice
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg clone public bob
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
Once again we have to configure their repositories: enable <tt class="docutils literal"><span class="pre">evolve</span></tt> and (since Alice and Bob will be pulling directly from each other) make their repositories non-publishing. Edit Alice’s configuration:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R alice config --edit --local
</pre>
</div>
</div>
<p>
and add
</p>
<div class="highlight-python">
<div class="highlight">
<pre>[extensions]
rebase =
evolve =
[phases]
publish = false
</pre>
</div>
</div>
<p>
Then edit Bob’s repository configuration:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg -R bob config --edit --local
</pre>
</div>
</div>
<p>
and add the same text.
</p>
</div>
<div class="section" id="example-6-divergent-changesets">
<h3>
<a class="toc-backref" href="#id17">Example 6: Divergent changesets</a><a class="headerlink" href="#example-6-divergent-changesets" title="Permalink to this headline"></a>
</h3>
<p>
When an obsolete changeset has two successors, those successors are <em>divergent</em>. One way to get into such a situation is by failing to communicate with your teammates. Let’s see how that might happen.
</p>
<p>
First, we’ll have Bob commit a bug fix that could still be improved:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd bob
$ echo 'pretty good fix' &gt;&gt; file1
$ hg commit -u bob -m 'fix bug 24 (v1)' # rev 4:2fe6
</pre>
</div>
</div>
<p>
Since Alice and Bob are now in cowboy mode, Alice pulls Bob’s draft changeset and amends it herself.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../alice
$ hg pull -u ../bob
[...]
added 1 changesets with 1 changes to 1 files
$ echo 'better fix (alice)' &gt;&gt; file1
$ hg amend -u alice -m 'fix bug 24 (v2 by alice)'
</pre>
</div>
</div>
<p>
But Bob has no idea that Alice just did this. (See how important good communication is?) So he implements a better fix of his own:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../bob
$ echo 'better fix (bob)' &gt;&gt; file1
$ hg amend -u bob -m 'fix bug 24 (v2 by bob)' # rev 6:a360
</pre>
</div>
</div>
<p>
At this point, the divergence exists, but only in theory: Bob’s original changeset, 4:2fe6, is obsolete and has two successors. But those successors are in different repositories, so the trouble is not visible to anyone yet. It will be as soon as Bob pulls from Alice’s repository (or vice-versa).
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg pull ../alice
[...]
added 1 changesets with 1 changes to 2 files (+1 heads)
(run 'hg heads' to see heads, 'hg merge' to merge)
2 new divergent changesets
</pre>
</div>
</div>
<p>
Figure 9 shows the situation in Bob’s repository.
</p>
<blockquote>
<div>
[figure SG09: Bob’s repo with 2 heads for the 2 divergent changesets, 6:a360 and 7:e3f9; wc is at 6:a360; both are successors of obsolete 4:2fe6, hence divergence]
</div>
</blockquote>
<p>
Now we need to get out of trouble. As usual, the answer is to evolve history.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ HGMERGE=internal:other hg evolve
merge:[6] fix bug 24 (v2 by bob)
with: [7] fix bug 24 (v2 by alice)
base: [4] fix bug 24 (v1)
0 files updated, 1 files merged, 0 files removed, 0 files unresolved
</pre>
</div>
</div>
<p>
Figure 10 shows how Bob’s repository looks now.
</p>
<blockquote>
<div>
[figure SG10: only one visible head, 9:5ad6, successor to hidden 6:a360 and 7:e3f9]
</div>
</blockquote>
<p>
We carefully dodged a merge conflict by specifying a merge tool (<tt class="docutils literal"><span class="pre">internal:other</span></tt>) that will take Alice’s changes over Bob’s. (You might wonder why Bob wouldn’t prefer his own changes by using <tt class="docutils literal"><span class="pre">internal:local</span></tt>. He’s avoiding a <a class="reference internal" href="#bug">bug</a> in <tt class="docutils literal"><span class="pre">evolve</span></tt> that occurs when evolving divergent changesets using <tt class="docutils literal"><span class="pre">internal:local</span></tt>.)
</p>
<p>
# XXX this link does not work .. <span class="target" id="bug">bug</span>: <a class="reference external" href="https://bitbucket.org/marmoute/mutable-history/issue/48/">https://bitbucket.org/marmoute/mutable-history/issue/48/</a>
</p>
<p>
** STOP HERE: WORK IN PROGRESS **
</p>
</div>
<div class="section" id="phase-divergence-when-a-rewritten-changeset-is-made-public">
<h3>
<a class="toc-backref" href="#id18">Phase-divergence: when a rewritten changeset is made public</a><a class="headerlink" href="#phase-divergence-when-a-rewritten-changeset-is-made-public" title="Permalink to this headline"></a>
</h3>
<p>
If Alice and Bob are collaborating on some mutable changesets, it’s possible to get into a situation where an otherwise worthwhile changeset cannot be pushed to the public repository; it is <em>phase-divergent</em> with another changeset that was made public first. Let’s demonstrate one way this could happen.
</p>
<p>
It starts with Alice committing a bug fix. Right now, we don’t yet know if this bug fix is good enough to push to the public repository, but it’s good enough for Alice to commit.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd alice
$ echo 'fix' &gt; file2
$ hg commit -A -m 'fix bug 15'
adding file2
</pre>
</div>
</div>
<p>
Now Bob has a bad idea: he decides to pull whatever Alice is working on and tweak her bug fix to his taste:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../bob
$ hg pull -u ../alice
[...]
added 1 changesets with 1 changes to 1 files
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ echo 'Fix.' &gt; file2
$ hg amend -A -m 'fix bug 15 (amended)'
</pre>
</div>
</div>
<p>
(Note the lack of communication between Alice and Bob. Failing to communicate with your colleagues is a good way to get into trouble. Nevertheless, <tt class="docutils literal"><span class="pre">evolve</span></tt> can usually sort things out, as we will see.)
</p>
<blockquote>
<div>
[figure SG06: Bob’s repo with one amendment]
</div>
</blockquote>
<p>
After some testing, Alice realizes her bug fix is just fine as it is: no need for further polishing and amending, this changeset is ready to publish.
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../alice
$ hg push
[...]
added 1 changesets with 1 changes to 1 files
</pre>
</div>
</div>
<p>
This introduces a contradiction: in Bob’s repository, changeset 2:e011 (his copy of Alice’s fix) is obsolete, since Bob amended it. But in Alice’s repository (and the <tt class="docutils literal"><span class="pre">public</span></tt> repository), that changeset is public: it is immutable, carved in stone for all eternity. No changeset can be both obsolete and public, so Bob is in for a surprise the next time he pulls from <tt class="docutils literal"><span class="pre">public</span></tt>:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ cd ../bob
$ hg pull -q -u
1 new phase-divergent changesets
</pre>
</div>
</div>
<p>
Figure 7 shows what just happened to Bob’s repository: changeset 2:e011 is now public, so it can’t be obsolete. When that changeset was obsolete, it made perfect sense for it to have a successor, namely Bob’s amendment of Alice’s fix (changeset 4:fe88). But it’s illogical for a public changeset to have a successor, so 4:fe88 is troubled: it has become <em>bumped</em>.
</p>
<blockquote>
<div>
[figure SG07: 2:e011 now public not obsolete, 4:fe88 now bumped]
</div>
</blockquote>
<p>
As usual when there’s trouble in your repository, the solution is to evolve it:
</p>
<div class="highlight-python">
<div class="highlight">
<pre>$ hg evolve --all
</pre>
</div>
</div>
<p>
Figure 8 illustrates Bob’s repository after evolving away the bumped changeset. Ignoring the obsolete changesets, Bob now has a nice, clean, simple history. His amendment of Alice’s bug fix lives on, as changeset 5:227d—albeit with a software-generated commit message. (Bob should probably amend that changeset to improve the commit message.) But the important thing is that his repository no longer has any troubled changesets, thanks to <tt class="docutils literal"><span class="pre">evolve</span></tt>.
</p>
<blockquote>
<div>
[figure SG08: 5:227d is new, formerly bumped changeset 4:fe88 now hidden]
</div>
</blockquote>
</div>
</div>
<div class="section" id="conclusion">
<h2>
<a class="toc-backref" href="#id19">Conclusion</a><a class="headerlink" href="#conclusion" title="Permalink to this headline"></a>
</h2>
<p>
Mutable history is a powerful tool. Like a sharp knife, an experienced user can do wonderful things with it, much more wonderful than with a dull knife (never mind a rusty spoon). At the same time, an inattentive or careless user can do harm to himself or others. Mercurial with <tt class="docutils literal"><span class="pre">evolve</span></tt> goes to great lengths to limit the harm you can do by trying to handle all possible types of “troubled” changesets. Nevertheless, having a first-aid kit nearby does not mean you should stop being careful with sharp knives.
</p>
<p>
Mutable history shared across multiple repositories by a single developer is a natural extension of this model. Once you are used to using a single sharp knife on its own, it’s pretty straightforward to chop onions and mushrooms using the same knife, or to alternate between two chopping boards with different knives.
</p>
<p>
Mutable history shared by multiple developers is a scary place to go. Imagine a professional kitchen full of expert chefs tossing their favourite knives back and forth, with the occasional axe or chainsaw thrown in to spice things up. If you’re confident that you <em>and your colleagues</em> can do it without losing a limb, go for it. But be sure to practice a lot first before you rely on it!
</p>
</div>
</div>
</div>
<div class="bottomnav">
<p>
«&#160;&#160;<a href="user-guide.html">Evolve: User Guide</a> &#160;&#160;::&#160;&#160; <a class="uplink" href="index.html">Contents</a> &#160;&#160;::&#160;&#160; <a href="concepts.html">Evolve: Concepts</a>&#160;&#160;»
</p>
</div>
<div class="footer">
© Copyright 2010-2014, Pierre-Yves David, Greg Ward, and contributors. Last updated on Sep 21, 2017. Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.3.
</div>
</body>
</html>

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/metadata-content-missing/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/metadata-content-missing/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "Creator Name",
"Direction": null,
"Excerpt": "Preferred description",
"Image": null,
"Title": "My title",
"SiteName": null
}

19
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/metadata-content-missing/expected.html vendored Normal file
View File

@ -0,0 +1,19 @@
<article>
<h2>Test document title</h2>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>

33
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/metadata-content-missing/source.html vendored Normal file
View File

@ -0,0 +1,33 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Title Element</title>
<meta property="x:title dc:title" content="My title"/>
<meta property="dc:creator twitter:site_name" content="Creator Name"/>
<meta name="author" content="FAIL"/>
<meta property="og:description twitter:description"/>
<meta property="dc:description og:description" content="Preferred description"/>
</head>
<body>
<article>
<h1>Test document title</h1>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<p>
Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>
</body>
</html>

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/normalize-spaces/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/normalize-spaces/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Lorem\n ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n\ttab here\n incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,\n quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo\n consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse\n cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non\n proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",
"Image": null,
"Title": "Normalize space test",
"SiteName": null
}

26
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/normalize-spaces/expected.html vendored Normal file
View File

@ -0,0 +1,26 @@
<article>
<h2>Lorem</h2>
<p>
Lorem
ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tab here
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
<h2>Foo</h2>
<p>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation
ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</p>
</article>

35
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/normalize-spaces/source.html vendored Normal file
View File

@ -0,0 +1,35 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Normalize space test</title>
</head>
<body>
<article>
<h1>Lorem</h1>
<div>
Lorem
ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tab here
incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
<h2>Foo</h2>
<div>
Tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam,
quis nostrud exercitation
ullamco laboris nisi ut aliquip ex ea commodo
consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse
cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non
proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
</div>
</article>
</body>
</html>

11
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-3/expected-images.json vendored Normal file
View File

@ -0,0 +1,11 @@
[
"https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/00winterutilities1-facebookJumbo.jpg",
"https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&auto=webp&disable=upscale",
"https:\/\/static01.nyt.com\/images\/2018\/02\/20\/multimedia\/author-corey-kilgannon\/author-corey-kilgannon-thumbLarge.jpg",
"https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilitiesOAK15\/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&auto=webp&disable=upscale",
"https:\/\/static01.nyt.com\/images\/2019\/02\/15\/nyregion\/00winterutilitiesOAK11\/merlin_94083158_9e622a52-ec2f-4fbd-845c-5d530e94bc82-articleLarge.jpg?quality=90&auto=webp",
"https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilitiesOAK13\/merlin_77720485_1f5be529-e659-49c3-a0bc-a3f312835d14-articleLarge.jpg?quality=90&auto=webp",
"https:\/\/static01.nyt.com\/images\/2017\/12\/07\/nyregion\/00winter4\/00winter4-articleLarge.jpg?quality=90&auto=webp",
"https:\/\/static01.nyt.com\/images\/2019\/02\/15\/nyregion\/00winterutilitiesOAK12\/merlin_77751796_7f1b7a7a-23e9-49b2-b12e-1edfed680911-articleLarge.jpg?quality=90&auto=webp",
"https:\/\/static01.nyt.com\/images\/2019\/02\/12\/nyregion\/00winterutilities2\/merlin_150504129_e893b874-01eb-4d8f-8158-18512153e414-articleLarge.jpg?quality=90&auto=webp"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-3/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "By Corey Kilgannon",
"Direction": null,
"Excerpt": "New York\u2019s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and \u201cfreeze-thaw\u201d cycles of winter make it even worse.",
"Image": "https:\/\/static01.nyt.com\/images\/2019\/02\/21\/nyregion\/21winterutilities1\/00winterutilities1-facebookJumbo.jpg",
"Title": "Manhole Fires and Burst Pipes: How Winter Wreaks Havoc on What\u2019s Underneath N.Y.C.",
"SiteName": null
}

267
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-3/expected.html vendored Normal file
View File

@ -0,0 +1,267 @@
<article id="story">
<header>
<p>
New York’s aging below-street infrastructure is tough to maintain, and the corrosive rock salt and “freeze-thaw” cycles of winter make it even worse.
</p>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemscope="itemscope" itemprop="associatedMedia" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-superJumbo.jpg?quality=90&amp;auto=webp 2048w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 80vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilities1/merlin_150498339_cf9085e5-9756-4169-a5a5-5b516316a3fa-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>A Con Edison worker repairing underground cables this month in Flushing, Queens. The likely source of the problem was water and rock salt that had seeped underground.</span><span itemprop="copyrightHolder"><span>Credit</span><span><span>Credit</span><span>Chang W. Lee/The New York Times</span></span></span>
</figcaption>
</figure>
</div>
<div>
<div>
<p><a href="https://www.nytimes.com/by/corey-kilgannon"><img alt="Corey Kilgannon" title="Corey Kilgannon" src="https://static01.nyt.com/images/2018/02/20/multimedia/author-corey-kilgannon/author-corey-kilgannon-thumbLarge.jpg"></a>
</p>
</div>
<ul>
<li>
<time datetime="2019-02-21">Feb. 21, 2019</time>
</li>
<li>
</li>
</ul>
</div>
</header>
<section name="articleBody" itemprop="articleBody">
<div>
<p>
<em>[What you need to know to start the day:</em> <a href="https://www.nytimes.com/newsletters/newyorktoday?module=inline" title><em>Get New York Today in your inbox</em></a><em>.]</em>
</p>
<p>
A series of recent <a href="https://nypost.com/2019/02/17/manhole-fires-force-theater-evacuations-in-midtown/" title rel="noopener noreferrer" target="_blank">manhole fires</a> in the heart of Manhattan forced the evacuation of several theaters and was a stark reminder that the subway is not the only creaky infrastructure beneath the streets of New York City.
</p>
<p>
Underground lies a chaotic assemblage of utilities that, much like the subway, are lifelines for the city: a sprawling tangle of water mains, power cables, gas and steam lines, telecom wires and sewers.
</p>
<p>
The city has one of the oldest and largest networks of subterranean infrastructure in the world, with some portions dating more than a century and prone to leaks and cracks.
</p>
<p>
And winter — from the corrosive rock salt used on streets and sidewalks to “freeze-thaw” cycles that weaken pipes — makes infrastructure problems even worse.
</p>
</div>
<div>
<p>
In the late 1800s, many of the city’s overhead utilities were buried to lessen the exposure to winter weather. “People think it’s all protected and safe, but it’s really not,” said Patrick McHugh, vice president of electrical engineering and planning for Con Edison, which maintains about 90,000 miles of underground cable in the city.
</p>
<p>
“You have water, sewage, electricity and gas down there, and people don’t appreciate the effort that goes into keeping all that working,” he added.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-superJumbo.jpg?quality=90&amp;auto=webp 2000w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 80vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK15/merlin_56025490_f9412a36-eeb9-4a10-a41e-f324eb7a3248-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>In the late 1800s, overhead utilities were buried to lessen the exposure to winter weather.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Kirsten Luce for The New York Times</span></span>
</figcaption>
</figure>
</div>
<div>
<h2 id="link-32de4094">
Rock salt on icy streets can cause mayhem below them
</h2>
<p>
When rock salt melts ice, and the water seeps down manholes and into electrical units, it can set off fires and explosions strong enough to pop a 300-pound manhole cover five stories into the air.
</p>
<p>
For days after a storm, Con Edison officials say, they often deal with scores of electrical fires caused by the rock salt eating away at electrical cable insulation. The wet salt can create sparking that burns the insulation, producing both fire and gases that can combust and pop the manhole lids.
</p>
<p>
To alleviate the threat, the officials said, the utility switched most of its manhole covers to vented ones that allow gases to escape, “so they cannot form a combustible amount,” Mr. McHugh said.
</p>
</div>
<div>
<p>
“It also lets smoke escape, which can tip off the public to notify the authorities,” he added.
</p>
<p>
Winter can also bring an increase in gas-line breakages. Con Edison, which maintains 4,300 miles of gas mains in and around New York City, records about 500 leaks — most of them nonemergencies — in a typical month, but many more in winter.
</p>
<p>
Even this past January, which was unseasonably mild, there were 750 leaks, Con Edison officials said.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK11/merlin_94083158_9e622a52-ec2f-4fbd-845c-5d530e94bc82-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>There are typically between 400 and 600 water main breaks each year in New York City, an official said.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Michael Appleton for The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK11/merlin_94083158_9e622a52-ec2f-4fbd-845c-5d530e94bc82-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-470a509">
‘Freeze-thaws’ linked to climate change can imperil utilities
</h2>
<p>
The extreme temperature swings that many researchers link to climate change are adding to the challenges of winter.
</p>
<p>
Officials monitor weather forecasts closely for freeze-thaw cycles, when they put extra repair crews on call.
</p>
<p>
During a polar vortex in late January, for instance, single-digit temperatures in the city quickly ballooned into the 50s. The thaw, much welcomed by many New Yorkers, worried Tasos Georgelis, deputy commissioner for water and sewer operations at the Department of Environmental Protection, which operates the city’s <a href="https://www.nytimes.com/interactive/2016/03/24/nyregion/how-nyc-gets-its-water-new-york-101.html?action=click&amp;contentCollection=New%20York&amp;region=Footer&amp;module=inline&amp;version=WhatsNext&amp;contentID=WhatsNext&amp;moduleDetail=undefined&amp;pgtype=Multimedia" title>water system</a>.
</p>
<p>
“When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” Mr. Georgelis said.
</p>
<p>
Along the city’s roughly 6,500 miles of water mains, there are typically between 400 and 600 breaks a year, he added. The majority occur in winter, when the cold can make older cast-iron mains brittle.
</p>
</div>
<div>
<p>
Environmental Protection officials said the department repaired 75 water-main breaks in January, including one in Lower Manhattan that disrupted rush-hour subway service and another on the West Side that snarled traffic and left nearby buildings without water for hours.
</p>
<p>
The city’s 7,500 miles of sewer lines are less affected by cold weather because they are generally buried deeper than other utilities, below the frost line, agency officials said.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK13/merlin_77720485_1f5be529-e659-49c3-a0bc-a3f312835d14-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>In 1978, a water main break caused severe flooding in Bushwick, Brooklyn.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Fred R. Conrad/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/21/nyregion/21winterutilitiesOAK13/merlin_77720485_1f5be529-e659-49c3-a0bc-a3f312835d14-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-4c678ca">
How to replace century-old utilities: slowly, and carefully
</h2>
<p>
Upgrading the city’s below-street utilities is a slow, painstaking process, “because you have such a fixed-in-place system,” said Rae Zimmerman, a research professor of planning and public administration at New York University.
</p>
<p>
But there is progress. Con Edison officials said they had begun replacing the city’s nearly 1,600 miles of natural gas lines — which were made of either cast iron or unprotected steel — with plastic piping. The plastic is less susceptible to corrosion, cracks and leaks, said the officials, who added that they were swapping about 100 miles of line each year.
</p>
<p>
The city is also replacing older, leak-prone water and sewer mains.
</p>
<p>
Some pipes that are more than a century old hold up because they were built with a thicker grade of cast iron, according to Environmental Protection Department officials. For less healthy ones, the agency has invested more than $1 billion in the past five years — with an additional $1.4 billion budgeted over the next five years — for upgrades and replacements. New pipes will be made of a more durable, graphite-rich cast iron known as ductile iron.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2017/12/07/nyregion/00winter4/00winter4-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>Matt Cruz snowboarded through Manhattan’s Lower East Side after a snowstorm in 2016 left the streets coated in slush and rock salt.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Hiroko Masuike/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2017/12/07/nyregion/00winter4/00winter4-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-132c3a6b">
Construction during the winter isn’t always a good idea
</h2>
<p>
Of course, winter also poses problems aboveground. Most nonemergency repair and construction work involving concrete is halted because concrete and some types of dirt, used to fill in trenches, freeze in colder temperatures, said Ian Michaels, a spokesman for the city’s Department of Design and Construction.
</p>
<p>
Digging by hand is also a challenge in frozen ground, so many excavations that are close to pipes and other utilities are put off, Mr. Michaels said.
</p>
</div>
<div>
<p>
And asphalt is harder to obtain because it must be kept and transported at high temperatures, he added.
</p>
<p>
In the extreme cold, city officials will not risk shutting down water mains for construction because spillage into the street could freeze, Mr. Michaels said. He added that stopping the water flow could freeze the private water-service connections that branch off the mains, he said.
</p>
<p>
Even the basic task of locating utilities under the street can be complicated because infrastructure has been added piecemeal over the decades.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK12/merlin_77751796_7f1b7a7a-23e9-49b2-b12e-1edfed680911-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>A water main break in Manhattan in 2014. “When you get a freeze and a thaw, the ground around the water mains expands and contracts, and puts external pressure on the pipes,” said Tasos Georgelis of the city's Department of Environmental Protection.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Ãngel Franco/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/15/nyregion/00winterutilitiesOAK12/merlin_77751796_7f1b7a7a-23e9-49b2-b12e-1edfed680911-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-7ba21106">
Almost 1.8 million potholes filled since de Blasio took office
</h2>
<p>
Street surfaces are affected by winter weather, too: Last year, the city filled 255,904 potholes.
</p>
<p>
And should anyone forget that filling potholes, like snow removal, is a sacred staple of constituent services, transportation officials have compiled the number of potholes the city has filled — more than 1,786,300 — since Mayor Bill de Blasio took office in 2014.
</p>
<p>
Potholes form when water and salt seep into cracks, freeze and expand, creating a larger crevice, said Joe Carbone, who works for the Transportation Department, where he is known as the pothole chief.
</p>
<p>
Simply put, more freeze-thaw cycles <a href="https://www.nytimes.com/2011/02/07/nyregion/07pothole.html?module=inline" title>result in more potholes</a>, he said. Currently, the department has 25 crews repairing potholes. During peak pothole-repair season in early March, that number can expand to more than 60.
</p>
</div>
<div>
<p>
Still, the department is continually resurfacing the city’s more than 6,000 miles of streets and 19,000 lane miles. Each year, agency officials said, it uses more than one million tons of asphalt to repave more than 1,300 lane-miles of street.
</p>
</div>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemprop="associatedMedia" itemscope="itemscope" itemid="https://static01.nyt.com/images/2019/02/12/nyregion/00winterutilities2/merlin_150504129_e893b874-01eb-4d8f-8158-18512153e414-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<div>
<p><span>Image</span></p>
</div>
<figcaption itemprop="caption description">
<span>Workers learning how to fix water main breaks at a training center in Queens.</span><span itemprop="copyrightHolder"><span>Credit</span><span>Chang W. Lee/The New York Times</span></span>
</figcaption>
<img src="https://static01.nyt.com/images/2019/02/12/nyregion/00winterutilities2/merlin_150504129_e893b874-01eb-4d8f-8158-18512153e414-articleLarge.jpg?quality=90&amp;auto=webp"></figure>
</div>
<div>
<h2 id="link-13d0a396">
So, where do people learn how to fix some of these issues?
</h2>
<p>
Of the 400 city laborers who work on water mains, many learn the finer points of leak repair at a training center in Queens, where underground pipes are made to spring leaks for repair drills.
</p>
<p>
Workers from the Department of Environmental Protection recently gathered around a muddy hole as a co-worker, Nehemiah Dejesus, scrambled to apply a stainless-steel repair clamp around a cracked segment that was spewing water.
</p>
<p>
“Don’t get nervous,” instructed Milton Velez, the agency’s district supervisor for Queens.
</p>
<p>
“I’m not,” Mr. Dejesus said as he secured the clamp and stopped the leak. “It’s ‘Showtime at the Apollo.’”
</p>
</div>
</section>
<div>
<div>
<p>
Corey Kilgannon is a Metro reporter covering news and human interest stories. His writes the <a href="https://www.nytimes.com/column/character-study">Character Study</a> column in the Sunday Metropolitan section. He was also part of the team that won the 2009 Pulitzer Prize for Breaking News. <span><a href="https://twitter.com/coreykilgannon" rel="noopener noreferrer" target="_blank"><span>@</span>coreykilgannon</a> <span>•</span> <a href="https://www.facebook.com/corey.kilgannon.9" rel="noopener noreferrer" target="_blank">Facebook</a></span>
</p>
</div>
<div><p>
A version of this article appears in print on </p><p>, on Page </p><p>A</p><p>22</p><p> of the New York edition</p><p> with the headline: </p><p>Under the City’s Streets, A Battle Against Winter<span>. <a href="http://www.nytreprints.com/">Order Reprints</a> | <a href="http://www.nytimes.com/pages/todayspaper/index.html">Today’s Paper</a> | <a href="https://www.nytimes.com/subscriptions/Multiproduct/lp8HYKU.html?campaignId=48JQY">Subscribe</a></span>
</p></div>
</div>
</article>

1638
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-3/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

4
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-4/expected-images.json vendored Normal file
View File

@ -0,0 +1,4 @@
[
"https:\/\/static01.nyt.com\/images\/2018\/09\/15\/business\/15DEBTS01\/15DEBTS01-facebookJumbo.jpg",
"https:\/\/static01.nyt.com\/images\/2018\/09\/15\/business\/15DEBTS01\/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=75&auto=webp&disable=upscale"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-4/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "By Nelson D. Schwartz",
"Direction": null,
"Excerpt": "Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.",
"Image": "https:\/\/static01.nyt.com\/images\/2018\/09\/15\/business\/15DEBTS01\/15DEBTS01-facebookJumbo.jpg",
"Title": "As Debt Rises, the Government Will Soon Spend More on Interest Than on the Military",
"SiteName": null
}

214
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-4/expected.html vendored Normal file
View File

@ -0,0 +1,214 @@
<article id="story">
<header>
<p>
Tax cuts, spending increases and higher interest rates could make it harder to respond to future recessions and deal with other needs.
</p>
<div data-testid="photoviewer-wrapper">
<figure aria-label="media" role="group" itemscope="itemscope" itemprop="associatedMedia" itemid="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=90&amp;auto=webp" itemtype="http://schema.org/ImageObject">
<p><span>Image</span><img alt src="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale" srcset="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=90&amp;auto=webp 600w,https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-jumbo.jpg?quality=90&amp;auto=webp 1024w,https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-superJumbo.jpg?quality=90&amp;auto=webp 2048w" sizes="((min-width: 600px) and (max-width: 1004px)) 84vw, (min-width: 1005px) 60vw, 100vw" itemprop="url" itemid="https://static01.nyt.com/images/2018/09/15/business/15DEBTS01/merlin_138209730_5f3b5746-4962-4207-a24f-aea644a8636f-articleLarge.jpg?quality=75&amp;auto=webp&amp;disable=upscale">
</p>
<figcaption itemprop="caption description">
<span>Interest payments on the federal debt could surpass the Defense Department budget in 2023.</span><span itemprop="copyrightHolder"><span>Credit</span><span><span>Credit</span><span>Jeon Heon-Kyun/EPA, via Shutterstock</span></span></span>
</figcaption>
</figure>
</div>
</header>
<section name="articleBody" itemprop="articleBody">
<div>
<p>
The federal government could soon pay more in interest on its debt than it spends on the military, Medicaid or children’s programs.
</p>
<p>
The run-up in borrowing costs is a one-two punch brought on by the need to finance a fast-growing budget deficit, worsened by tax cuts and steadily rising interest rates that will make the debt more expensive.
</p>
<p>
With less money coming in and more going toward interest, political leaders will find it harder to address pressing needs like fixing crumbling roads and bridges or to make emergency moves like pulling the economy out of future recessions.
</p>
<p>
Within a decade, more than $900 billion in interest payments will be due annually, easily outpacing spending on myriad other programs. Already the fastest-growing major government expense, the cost of interest is on track to hit $390 billion next year, nearly 50 percent more than in 2017, according to the Congressional Budget Office.
</p>
</div>
<div>
<p>
“It’s very much something to worry about,” said C. Eugene Steuerle, a fellow at the Urban Institute and a co-founder of the Urban-Brookings Tax Policy Center in Washington. “Everything else is getting squeezed.”
</p>
<p>
Gradually rising interest rates would have made borrowing more expensive even without additional debt. But the tax cuts passed late last year have created a deeper hole, with the deficit increasing faster than expected. A budget bill <a href="https://www.nytimes.com/2018/02/08/us/politics/congress-budget-deal-vote.html?module=inline" title>approved in February that raised</a> spending by $300 billion over two years will add to the financial pressure.
</p>
<p>
The deficit is expected to total nearly $1 trillion next year — the first time it has been that big since 2012, when the economy was still struggling to recover from the financial crisis and interest rates were near zero.
</p>
</div>
<div>
<p>
Deficit hawks have gone silent, even proposing changes that would exacerbate the deficit. House Republicans introduced legislation this month that would make the tax cuts permanent.
</p>
<p>
“The issue has just disappeared,” said Senator Mark Warner, a Virginia Democrat. “There’s collective amnesia.”
</p>
</div>
<div>
<p>
The combination, say economists, marks a journey into mostly uncharted financial territory.
</p>
<p>
In the past, government borrowing expanded during recessions and waned in recoveries. That countercyclical policy has been a part of the standard Keynesian toolbox to combat downturns since the Great Depression.
</p>
<p>
The deficit is soaring now as the economy booms, meaning the stimulus is pro-cyclical. The risk is that the government would have less room to maneuver if the economy slows.
</p>
</div>
<div>
<p>
Aside from wartime or a deep downturn like the 1930s or 2008-9, “this sort of aggressive fiscal stimulus is unprecedented in U.S. history,” said Jeffrey Frankel, an economist at Harvard.
</p>
<p>
Pouring gasoline on an already hot economy has resulted in faster growth — the economy expanded at an annualized rate of 4.2 percent in the second quarter. But Mr. Frankel warns that when the economy weakens, the government will find it more difficult to cut taxes or increase spending.
</p>
<p>
Lawmakers might, in fact, feel compelled to cut spending as tax revenue falls, further depressing the economy. “There will eventually be another recession, and this increases the chances we will have to slam on the brakes when the car is already going too slowly,” Mr. Frankel said.
</p>
<h2 id="link-cb5ee12">
Interest costs make it harder for the government to do other things
</h2>
</div>
<div>
<p>
Finding the money to pay investors who hold government debt will crimp other parts of the budget. In a decade, interest on the debt will eat up 13 percent of government spending, up from 6.6 percent in 2017.
</p>
<p>
“By 2020, we will spend more on interest than we do on kids, including education, food stamps and aid to families,” said Marc Goldwein, senior policy director at the Committee for a Responsible Federal Budget, a research and advocacy organization.
</p>
</div>
<div>
<p>
Interest costs already dwarf spending on many popular programs. For example, grants to students from low-income families for college total roughly $30 billion — about one-tenth of what the government will pay in interest this year. Interest payments will overtake Medicaid in 2020 and the Department of Defense budget in 2023.
</p>
<p>
What’s more, the heavy burden of interest payments could make it harder for the government to repair aging infrastructure or take on other big new projects.
</p>
<p>
Mr. Trump has called for spending $1 trillion on infrastructure, but Congress has not taken up that idea.
</p>
</div>
<div>
<p>
More about the federal debt and the economy
</p>
</div>
<div>
<h2 id="link-1a1f1f7a">
The U.S. hasn’t faced this issue for years
</h2>
<p>
Until recently, ultralow interest rates, set by the Federal Reserve to support the economy, allowed lawmakers to borrow without fretting too much about the cost of that debt.
</p>
<p>
But as the economy has strengthened, the Fed has gradually raised rates, starting in December 2015. The central bank is expected to push rates up again on Wednesday, and more increases are in store.
</p>
<p>
“When rates went down to record lows, it allowed the government to take on more debt without paying more interest,” Mr. Goldwein said. “That party is ending.”
</p>
</div>
<div>
<p>
Since the beginning of the year, the yield on the 10-year Treasury note has risen by more than half a percentage point, to 3.1 percent. The Congressional Budget Office estimates that the yield will climb to 4.2 percent in 2021. Given that the total public debt of the United States stands at nearly $16 trillion, even a small uptick in rates can cost the government billions.
</p>
</div>
<div>
<p>
There’s no guarantee that these forecasts will prove accurate. If the economy weakens, rates might fall or rise only slightly, reducing interest payments. But rates could also overshoot the budget office forecast.
</p>
<p>
Some members of Congress want to set the stage for even more red ink. Republicans in the House want to make last year’s tax cuts permanent, instead of letting some of them expire at the end of 2025. That would reduce federal revenue by an additional $631 billion over 10 years, according to the <a href="https://www.taxpolicycenter.org/publications/analysis-protecting-family-and-small-business-tax-cuts-act-2018" title rel="noopener noreferrer" target="_blank">Tax Policy Center</a>.
</p>
<h2 id="link-7b39a7bd">
No, the United States isn’t at risk of becoming the next Greece
</h2>
</div>
<div>
<p>
Deficit hawks have warned for years that a day of reckoning is coming, exposing the United States to the kind of economic crisis that overtook profligate borrowers in the past like Greece or Argentina.
</p>
<p>
But most experts say that isn’t likely because the dollar is the world’s reserve currency. As a result, the United States still has plenty of borrowing capacity left because the Fed can print money with fewer consequences than other central banks.
</p>
<p>
And interest rates plunged over the last decade, even as the government turned to the market for trillions each year after the recession. That’s because Treasury bonds are still the favored port of international investors in any economic storm.
</p>
<p>
“We exported a financial crisis a decade ago, and the world responded by sending us money,” said William G. Gale, a senior fellow at the Brookings Institution.
</p>
<p>
But that privileged position has allowed politicians in both parties to avoid politically painful steps like cutting spending or raising taxes.
</p>
</div>
<div>
<p>
That doesn’t mean rapidly rising interest costs and a bigger deficit won’t eventually catch up with us.
</p>
<p>
Charles Schultze, chairman of the Council of Economic Advisers in the Carter administration, once summed up the danger of deficits with a metaphor. “It’s not so much a question of the wolf at the door, but termites in the woodwork.”
</p>
<h2 id="link-12df6cc7">
But Washington doesn’t want to hear about the potential problems
</h2>
<p>
Rather than simply splitting along party lines, lawmakers’ attitudes toward the deficit also depend on which party is in power. Republicans pilloried the Obama administration for proposing a large stimulus in the depths of the recession in 2009 and complained about the deficit for years.
</p>
<p>
In 2013, Senator Mitch McConnell of Kentucky called the debt and deficit “the transcendent issue of our era.” By 2017, as Senate majority leader, he quickly shepherded the tax cut through Congress.
</p>
<p>
Senator James Lankford, an Oklahoma Republican who warned of the deficit’s dangers in the past, nevertheless played down that threat on the Senate floor as the tax billed neared passage.
</p>
<p>
“I understand it’s a risk, but I think it’s an appropriate risk to be able to say let’s allow Americans to keep more of their own money to invest in this economy,” he said.
</p>
<p>
He also claimed the tax cuts would pay for themselves even as the Congressional Budget Office estimated that they would add $250 billion to the deficit on average from 2019 to 2024.
</p>
</div>
<div>
<p>
In an interview, Mr. Lankford insisted that the jury was still out on whether the tax cuts would generate additional revenue, citing the strong economic growth recently.
</p>
<p>
While the Republican about-face has been much more striking, Democrats have adjusted their position, too.
</p>
<p>
Mr. Warner, the Virginia Democrat, called last year’s tax bill “the worst piece of legislation we have passed since I arrived in the Senate.” In 2009, however, when Congress passed an $800 billion stimulus bill backed by the Obama administration, he called it “a responsible mix of tax cuts and investments that will create jobs.”
</p>
<p>
The difference, Mr. Warner said, was that the economy was near the precipice then.
</p>
<p>
“There was virtual unanimity among economists that we needed a stimulus,” he said. “But a $2 trillion tax cut at the end of a business cycle with borrowed money won’t end well.”
</p>
</div>
</section>
<div>
<div>
<p>
Nelson D. Schwartz has covered economics since 2012. Previously, he wrote about Wall Street and banking, and also served as European economic correspondent in Paris. He joined The Times in 2007 as a feature writer for the Sunday Business section. <span><a href="https://twitter.com/NelsonSchwartz" rel="noopener noreferrer" target="_blank"><span>@</span>NelsonSchwartz</a></span>
</p>
</div>
<div><p>
A version of this article appears in print on </p><p>, on Page </p><p>A</p><p>1</p><p> of the New York edition</p><p> with the headline: </p><p>What May Soon Exceed Cost of U.S. Military? Interest on U.S. Debt <span>. <a href="http://www.nytreprints.com/">Order Reprints</a> | <a href="http://www.nytimes.com/pages/todayspaper/index.html">Today’s Paper</a> | <a href="https://www.nytimes.com/subscriptions/Multiproduct/lp8HYKU.html?campaignId=48JQY">Subscribe</a></span>
</p></div>
</div>
</article>

3896
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/nytimes-4/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/remove-aria-hidden/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/remove-aria-hidden/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod\n tempor incididunt ut labore et dolore magna aliqua.",
"Image": null,
"Title": "Remove aria-hidden elements test",
"SiteName": null
}

7
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/remove-aria-hidden/expected.html vendored Normal file
View File

@ -0,0 +1,7 @@
<div>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
</div>

19
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/remove-aria-hidden/source.html vendored Normal file
View File

@ -0,0 +1,19 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>Remove aria-hidden elements test</title>
</head>
<body>
<article>
<h1>Lorem</h1>
<div>
<p><span aria-hidden="true">**WRONG**</span>Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod
tempor incididunt ut labore et dolore magna aliqua.</p>
<p>Ut enim ad minim veniam,
quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo
consequat.</p>
</div>
</article>
</body>
</html>

4
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/seattletimes-1/expected-images.json vendored Normal file
View File

@ -0,0 +1,4 @@
[
"https:\/\/static.seattletimes.com\/wp-content\/uploads\/2019\/04\/120028-1200x630.jpg",
"https:\/\/static.seattletimes.com\/wp-content\/uploads\/2019\/04\/120109-1020x680.jpg"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/seattletimes-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": "April 28, 2019 at 6:01 am Updated April 29, 2019 at 3:33 pm",
"Direction": null,
"Excerpt": "The story of Whole Foods\u2019 halibut deal opens a window into Amazon\u2019s grocery strategy and draws a line from a Seattle industry with roots in the 19th century to the dominant economic force of the 21st.",
"Image": "https:\/\/static.seattletimes.com\/wp-content\/uploads\/2019\/04\/120028-1200x630.jpg",
"Title": "Alaskan halibut, caught by a century-old Seattle boat, provides a glimpse of Amazon\u2019s strategy with Whole Foods",
"SiteName": "The Seattle Times"
}

179
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/seattletimes-1/expected.html vendored Normal file
View File

@ -0,0 +1,179 @@
<div itemprop="articleBody" id="article-content">
<p>
From the deck of his 106-year-old halibut schooner, undergoing a seasonal overhaul at Fisherman’s Terminal in Seattle, skipper Wade Bassi has better insight than most into what’s happening at Amazon-owned Whole Foods Market, at least as pertains to the product he knows best.
</p>
<p>
While he doesn’t buy halibut much — he’s got a freezer full of it — Bassi, 43 years a fisherman, keeps an eye on how it’s handled and presented in the grocery stores and fish markets.
</p>
<p>
“When you look at nice halibut, I mean it is pure white,” he said. “And it is flaky-looking, and it is beautiful. It’s translucent. If you’ve got that in the fish market, people are going to buy it.”
</p>
<p>
A few days earlier, Whole Foods touted a rarely seen promotional price for halibut as part of its ongoing campaign to revise the grocery chain’s high-cost reputation while maintaining its image for quality and sustainability.
</p>
<p>
“Whole Foods is one of the better ones, to be honest with you,” Bassi said. “But you know, Whole Foods, whole paycheck. … They usually do charge way more for everything than anywhere else. Which really surprises me that they’re selling this for $16-something a pound, because they’re not making anything on it.”
</p>
<p>
Whole Foods’ halibut deal opens a window into Amazon’s grocery strategy as it seeks to combine the defining characteristics of each brand, leverage its juggernaut Prime membership program and take a larger share of the grocery business from competitors such as Walmart, Kroger and Costco.
</p>
<p>
It also draws a long line from a major Seattle industry with roots in the 19<sup>th</sup> century to the dominant economic force of the 21<sup>st</sup>.
</p>
<p>
Amazon bought Whole Foods in August 2017 for $13.7 billion, its largest acquisition and an aggressive move into the grocery business.
</p>
<p>
The combination of the two has been steady, said Tom Forte, who follows Amazon as a managing director at the D.A. Davidson brokerage. In a few more years, he said, “You won’t recognize the original Whole Foods.”
</p>
<p>
Within months of the acquisition, Forte said, Whole Foods was selling cheaper cage-free eggs and organic ground beef, prices it said were a result of the deal.
</p>
<p>
Then came the integration of Prime, Amazon’s $119-a-year shipping and media-subscription program, which Amazon founder Jeff Bezos said last year had surpassed 100 million members. In Whole Foods, Prime operates as a hybrid of the customer-loyalty discount programs offered by most grocers — in which consumers trade details of their purchasing habits for lower prices — and a paid membership like at Costco or Sam’s Club.
</p>
<p>
Whole Foods stores have been festooned with yellow and blue signs pointing out Prime member benefits, one of which was fresh halibut fillets priced at $16.99 a pound, albeit only for a week earlier this month.
</p>
<p>
“I was shocked to see that level,” said Tyler Besecker, president of Mercer Island-based Dana F. Besecker Company, the largest buyer of Pacific halibut. The price, which was matched at Kroger-owned QFC stores in the region last week, is “as low as I’ve ever seen.” (Besecker does not currently supply Whole Foods.)
</p>
<p>
Fresh halibut fillets routinely sell for $24 to $28 a pound, and often more.
</p>
<p>
He said there’s little if any room for a profit at the promotional price offered by Whole Foods and QFC. “They might be selling those at cost or as loss leaders just to get people into the stores,” Besecker said.
</p>
<p>
In the competitive grocery business, promotions like this happen all the time. The thinking is that shoppers will be attracted by the discount on a staple or a prestige item, and then fill their carts with other groceries sold at a profit.
</p>
<p>
A Whole Foods spokeswoman declined to comment on pricing. The temporary halibut discount is one of more than 300 such Prime promotions Whole Foods plans in the next few months. The company also said it was lowering prices across the store, its third such announcement since the acquisition.
</p>
<p>
At the seafood counter in the Whole Foods store on Westlake, surrounded by Amazon headquarters buildings, a sign advertised “First of the Season Fresh Alaskan Halibut” and sported the blue Marine Stewardship Council (MSC) Certified Sustainable Seafood label.
</p>
<p>
Whole Foods has been a pioneer in sustainable-seafood marketing, beginning in 1999 when it began to stock fish with the MSC label. In the mid-2000s, Pacific halibut fishermen sought the certification — a system of third-party audits that tracks seafood from catch to market — and Whole Foods was there from the beginning.
</p>
<p>
“They were the first ones to market the MSC halibut,” said Bob Alverson, head of the Fishing Vessel Owners Association, representing boats that catch halibut and black cod and a driver of the certification effort. “It turned into quite a marketing advantage. Whole Foods saw that early. They were focusing on sustainable, high-quality food products. They had quite a bit of foresight, I think, in that direction.”
</p>
<p>
The certification comes with added costs borne by the fishermen and buyers, and passed on to consumers. But it’s also an assurance “that people are watching out for the resource,” he said.
</p>
<p>
As it tries to convince people it has lower prices, Whole Foods has been very careful to maintain the reputation built on products like MSC-certified halibut.
</p>
<div>
<figure id="image-11519494">
<img data-ratio="1.5" data-caption="Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)" class alt="Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)" src="https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1020x680.jpg" srcset="https://static.seattletimes.com/wp-content/uploads/2019/04/120109-300x200.jpg 300w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-768x512.jpg 768w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1024x683.jpg 1024w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-780x520.jpg 780w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1020x680.jpg 1020w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1560x1040.jpg 1560w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-375x250.jpg 375w" sizes="(max-width: 767px) calc(100vw - 20px), (max-width: 1019px) calc(100vw - 30px), (max-width: 1044px) calc(100vw - 60px), 970px" data-sizes="(max-width: 767px) calc(100vw - 20px), (max-width: 1019px) calc(100vw - 30px), (max-width: 1044px) calc(100vw - 60px), 970px" data-old-src="https://www.seattletimes.com/wp-content/themes/st_refresh/img/lazy-loading-14x9.png" data-src="https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1020x680.jpg" data-srcset="https://static.seattletimes.com/wp-content/uploads/2019/04/120109-300x200.jpg 300w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-768x512.jpg 768w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1024x683.jpg 1024w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-780x520.jpg 780w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1020x680.jpg 1020w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-1560x1040.jpg 1560w, https://static.seattletimes.com/wp-content/uploads/2019/04/120109-375x250.jpg 375w">
<figcaption>
<span>Amazon-owned Whole Foods touted a price cut on halibut as part of an announcement recently about lower prices on hundreds of items. (Ellen M. Banner / The Seattle Times)</span>
</figcaption>
</figure>
</div>
<p>
<strong>Whole Foods future</strong>
</p>
<p>
New signs in stores appeared this month, spelling out the value proposition it’s trying to strike: “New lower prices. Same high standards,” reads one, against a background image of carrots.
</p>
<p>
At the same time, the company claims its new prices and Prime deals have saved customers “hundreds of millions of dollars” since the Amazon acquisition.
</p>
<p>
If prices are being lowered and the quality bar stays the same, something else has to give.
</p>
<p>
Analysts provided a few theories:
</p>
<p>
Amazon could be willing to accept losses or slimmer profits within Whole Foods, as it has done in other businesses, in an effort to expand its customer base.
</p>
<p>
“Do they take the profit from their non-retail efforts, which today is primarily cloud computing, and then reinvest those profits to take share in grocery?” Forte said.
</p>
<p>
That could eventually open up an avenue to growth as the rate of expansion slows in Amazon’s broader U.S. retail sales.
</p>
<p>
“That’s why they need grocery to work,” Forte said. “Grocery’s a very big category.”
</p>
<p>
Whole Foods could also <span data-st-annotation-ref="1792ba">press suppliers to reduce their prices</span>, essentially cutting their profit.
</p>
<p>
“The worry with that is that it puts the squeeze on the producer upstream,” said Ananth Iyer, a professor at Purdue University whose research includes sustainability in supply chains. If producers are squeezed too much, he noted, they may start to cut corners.
</p>
<p>
So far, there’s no evidence this is happening in halibut, where fishery practices have been carefully managed with a goal of sustainability for nearly a century. Also, Whole Foods does not yet have the scale as a buyer to dictate prices the way a company like Costco does.
</p>
<p>
Forte said that even if it did have such clout, this would be a risky strategy that would undermine the very attributes of the Whole Foods brand that make it most valuable.
</p>
<p>
Another theory is that Amazon could apply more of its technology and expertise in logistics to create supply-chain efficiencies that would maintain its profits while benefiting producers and consumers, particularly with perishable grocery products, Iyer said. This is part of the promise of the acquisition in the first place.
</p>
<p>
“That’s a powerful combination,” Forte said. “The sustainable, the organic, the healthier food — all those qualities of Whole Foods, with the supply-chain technology of Amazon. It plays to the strengths of both sides.”
</p>
<p>
Forte said he expects Amazon to continue its aggressive moves on grocery pricing at Whole Foods. But he wonders when the price cuts will be broader, particularly as Amazon competes with the likes of Walmart and Kroger for a bigger slice of U.S. food and beverage retail sales, which totaled $726 billion in 2017. Whole Foods said it has lowered prices on hundreds of items, with an emphasis on fresh produce.
</p>
<p>
Forte described his attempt after the acquisition to get the ingredients for Rice Krispies Treats at Whole Foods. It was perhaps doomed from the start: Whole Foods doesn’t carry Rice Krispies. But he found an organic brown rice puff cereal and organic marshmallows. They were “so wildly expensive that we didn’t finish the exercise. I took the kids to Walmart and bought the ingredients for a pittance,” Forte said.
</p>
<p>
That points to the bigger question of how Amazon plans to position Whole Foods for the long term in its expanding array of physical retail-grocery formats. It now has 11 automated Go convenience stores and is rumored to be planning a new, low-priced grocery chain of its own that may deploy the same cashierless checkout technology.
</p>
<p>
Meanwhile, Whole Foods is not opening any new 365 stores, a smaller, lower-priced version of the main brand highlighting the company’s private-label products. Whole Foods co-founder and CEO John Mackey said in an internal memo that the “price distinction between the two brands has become less relevant” as Whole Foods lowered its prices, Yahoo Finance <a href="https://finance.yahoo.com/news/amazonowned-whole-foods-scraps-smaller-365-store-expansion-203543290.html" target="_blank">reported</a> earlier this year.
</p>
<p>
Amazon, too, is finding success with a growing stable of private-label brands — it had more than 100 as of last July, according to Coresight Research, double the number in 2017. One of these, Solimo, sells generic versions of everything from K-Cup coffee pods to Epsom salts to garbage bags and racked up more than $6 million in sales in January alone, according to data analysis firm 1010data.
</p>
<p>
Of course, there’s nothing generic about a “fresh, sustainable wild-caught halibut fillet.”
</p>
<p>
<strong>Back on the schooner</strong>
</p>
<p>
The day after Easter, Bassi and his crew — three family members and an unrelated father-and-son team — loaded up the Polaris, one of four century-old wooden schooners still chasing halibut out of Seattle. (The broader Washington-based halibut fleet numbers about 100 vessels.) Bassi’s father fished on the Polaris, which Bassi co-owns with&nbsp;Rolfe McCartney. Bassi’s grandfather fished halibut back when schooners carried small dories out to the fishing grounds, which made the landing of a fish that can grow to 500 pounds all the more exciting.
</p>
<p>
The Polaris motored out of Fisherman’s Terminal and through the Ballard Locks to begin the three-day journey through the Inside Passage to Ketchikan, Alaska. There, they take on tons of ice and bait, herring for the black cod Bassi will target first, and later chum salmon, codfish or octopus for the halibut.
</p>
<p>
From a base in Kodiak, Alaska, the Polaris makes a series of trips, at sea for a week or longer at a time, to fish as far away as Attu Island at the far western edge of the Aleutian Islands chain. “It’s a big range that we fish,” Bassi said.
</p>
<p>
The Polaris will trail long lines of hooks, leaving them to soak for several hours before reeling them in. The fish are stunned, bled and dressed, and put on ice in the hold. It is this fishing method that contributes to the quality of the halibut and sustainability of the fishery, as it reduces by-catch — the inadvertent taking of other <span data-st-annotation-ref="53071e">species</span>.
</p>
<p>
They negotiate to sell the fish with four or five buyers, such as Besecker, at a price that fluctuates throughout the season. This is a peak time of year for halibut, with consumers seeking out fresh fish for Easter and Mother’s Day, Besecker said. Fishermen are typically paid between $5 and $6 a pound for halibut.
</p>
<p>
The Alaskan halibut fishery has its troubles — as nearly all fisheries do — but has been rationalized and managed successfully, particularly over the last quarter-century. Alverson, a commissioner on the Seattle-based International Pacific Halibut Commission, which has managed catch limits for U.S. and Canadian fishermen since 1924, described it as a stable but declining resource.
</p>
<p>
This year, the <a href="https://iphc.int/management/fisheries" target="_blank">halibut fishery</a> in the Northern Pacific and Bering Sea is <a href="https://iphc.int/data/landings-2019" target="_blank">capped at 29.4 million pounds</a>, with most of that allocated to commercial fishing and smaller amounts reserved for recreational and tribal fishing, as well as by-catch of other commercial fisheries.
</p>
<p>
Alverson summed up the journey from a wooden fishing boat in the Bering Sea to the fish counter of a grocery store owned by a company that has redefined modern buying and selling: “It’s Seattle old school meets Seattle new school with Amazon.”
</p>
</div>

1659
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/seattletimes-1/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

3
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/topicseed-1/expected-images.json vendored Normal file
View File

@ -0,0 +1,3 @@
[
"https:\/\/topicseed.com\/static\/9c97da26f6eeee98fc2e628ca3416226\/57090\/content-depth-seo.png"
]

8
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/topicseed-1/expected-metadata.json vendored Normal file
View File

@ -0,0 +1,8 @@
{
"Author": null,
"Direction": null,
"Excerpt": "Content writers and marketers find it hard to write a lot of content about a very specific topic. They lose a lot of points on their content depth because they would rather focus on pushing thin content about plenty of topics.",
"Image": "https:\/\/topicseed.com\/static\/9c97da26f6eeee98fc2e628ca3416226\/57090\/content-depth-seo.png",
"Title": "Content Depth \u2014 Write Comprehensively About Your Core Topics",
"SiteName": "topicseed"
}

93
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/topicseed-1/expected.html vendored Normal file
View File

@ -0,0 +1,93 @@
<div>
<ul>
<li>
<a href="#assess-how-deep-is-your-content">Assess How Deep Is Your Content</a>
</li>
<li>
<a href="#rewrite-with-content-depth-in-mind">Rewrite With Content Depth In Mind</a>
</li>
<li>
<a href="#yes-content-depth-and-breadth-overlap">Yes, Content Depth and Breadth Overlap</a>
</li>
<li>
<a href="#depth-of-content--quality--frequency">Depth of Content = Quality + Frequency</a>
</li>
</ul>
<p>
<strong>Content depth</strong> is an arbitrary score or rating of how comprehensive the coverage of a specific topic is within a piece of content. <strong>Content breadth</strong> is an arbitrary grading of how many related subjects are you covering within your content.
</p>
<p>
And this distinction is important to make and establish from the beginning. Effective <a href="https://topicseed.com/blog/what-is-topical-authority" target="_blank" rel="nofollow noopener noreferrer"><strong>topical authority</strong></a> can only be gained when you use both content depth and content breadth in your overall content strategy for rapid search engine optimization gains. However, because most content writers prefer to write a little bit about many things rather than write a lot about one thing, you end up with a too little substance spread very thin.
</p>
<p>
Content depth should be the urgent priority for your content marketing strategy, and clearly defined in your <a href="http://fakehost/blog/content-briefs">content briefs</a>. Start by dominating your own core topics, before venturing across the pond and write about linked subject matters. Otherwise, you are the opposite of an authority as the definition states that an authority is <em>“a person with extensive or specialized knowledge about a subject; an expert”.</em> Lastly, do not mistake&nbsp;article depth vs. article length: a&nbsp;blog post’s extreme wordcount has nothing to do with its content depth.
</p>
<h2 id="assess-how-deep-is-your-content">
<a href="#assess-how-deep-is-your-content" aria-label="assess how deep is your content permalink"></a>Assess How Deep Is Your Content
</h2>
<p>
The first task on your list, right now, is to shortlist your core topics. What are you trying to be an expert on? Then, go through each one of your pieces of content and understand how well each blog post is covering&nbsp;its focus topic(s). Not how many times specific keywords appear, or how well the article is outlined and structured.
</p>
<p>
Put yourself in the shoes of an ignorant reader who seeks information. Read your article. <strong>And ask yourself how in-depth was the content you have written?</strong> I know the excuse you will come up with: this was written for beginners, therefore, it shouldn’t be too in-depth. And you are correct. Not every blog post is about absolute content depth otherwise we would only write one 10,000-word-long article, once and for all. But then, how well your beginner-level content pointing to your expert-level content?
</p>
<p>
In other words, each article should reach an incredible level of content depth for its expertise level. And then, provide further reading <em>(i.e. links)</em> to gain more knowledge, and depth. A lot of content editors write a beginner’s blog post and wait to see it perform well in order to write a more advanced sequel. Wrong. Give all the value so search engines can grade you highly on their authority scale for your core topics. Yes, it is a risk and you may write a dozen of articles on a specific topic that will never really rank at the top of SERPs, but <strong>reaching content depth is the first step towards SEO gains</strong>.
</p>
<p>
Remember that <strong>skyscraper content</strong> and <strong>10x content</strong> are not necessarily the answer. These content writing strategies state that in order to beat another piece of content, you need to write 10x more. Either in quantity with a 10x word count or in quality by putting times more information within your own piece of content. Such articles often become unreadable and discourage visitors from absorbing all the knowledge. The best alternative is the create <a href="https://topicseed.com/blog/how-broad-should-topics-be-for-pillar-pages" target="_blank" rel="nofollow noopener noreferrer">pillar pages</a> centered around core topics, and several articles dealing with each specific section in depth. This is <strong>deep content powered by a <a href="https://topicseed.com/blog/internal-linking-strategies-for-topic-clustering" target="_blank" rel="nofollow noopener noreferrer">smart internal linking strategy</a></strong>&nbsp;and search engines love that in this day and age where attention spans are short! <em>With that being said, avoid writing 600-word articles!</em>
</p>
<h2 id="rewrite-with-content-depth-in-mind">
<a href="#rewrite-with-content-depth-in-mind" aria-label="rewrite with content depth in mind permalink"></a>Rewrite With Content Depth In Mind
</h2>
<p>
Once you know which articles are lacking depth of knowledge and information, it is time to rethink each one. For each article, make a list of what essential pieces of information or data are missing. Then decide where to fit them, and decide whether the article would benefit from a full rewrite or not. As a rule of thumb, if you need to change a third of your article, you may need to rewrite it entirely. Of course, this does not mean erasing all work done prior, but it means starting afresh! Trying to <strong>fit deep content into an existing blog</strong> post gives you constraints so doing it from scratch can actually be easier to fight thin content.
</p>
<p>
As explained above, make sure you do not force yourself to write a much longer article to reach a <a href="https://moz.com/blog/blog-post-length-frequency" target="_blank" rel="nofollow noopener noreferrer">magic word count</a>. And if you do, it has to be natural. In many cases, articles written months or years ago may need some upkeeping: trimming the fat and removing parts that are not bringing much value. Replace these with your newer and deeper content.
</p>
<p>
All content writers know that when you open Google Docs, WordPress, or your text editor of choice, you will inevitably count your focus keywords’ frequency. Although I understand (yet question) the value of keywords in modern SEO, do not become obsessed with reaching a magic number for your keywords. No reader coming from Google is out there counting how often your keywords are appearing. And search engine algorithms will penalize you for writing for robots, rather than humans.
</p>
<p>
With the massive rise of voice searches, <a href="http://fakehost/blog/featured-snippets-using-questions">users tend to use full questions for their search queries</a>. What used to be <code>top bottled water brands</code>&nbsp;is now <code>OK google, what is the best bottled-water brand in Texas</code>?&nbsp;The point being, <a href="https://topicseed.com/blog/keyword-search-volume-overrated" target="_blank" rel="nofollow noopener noreferrer"><strong>keywords are losing traction</strong></a> to leave space for a more natural language understanding of a blog post’s textual content, and meaning.
</p>
<h2 id="yes-content-depth-and-breadth-overlap">
<a href="#yes-content-depth-and-breadth-overlap" aria-label="yes content depth and breadth overlap permalink"></a>Yes, Content Depth and Breadth Overlap
</h2>
<p>
<em>“A topic can be defined as the company it keeps.”</em> A very accurate saying loved by ontologists&nbsp;within the fields of&nbsp;computational linguistics, and information science. In simpler terms, a topic and all the terminology it is encompassing will inevitably overlap with related topics. Which, in turn, will form <a href="https://topicseed.com/blog/topic-clusters-relationships" target="_blank" rel="nofollow noopener noreferrer"><strong>topic clusters</strong></a>.
</p>
<p>
For example, it is obvious that despite being two different topics, <code>digital advertising</code>&nbsp;and <code>content marketing</code>&nbsp;share some common phrases and terms. Inevitably, a website picking one as its core topic will use words in some blog posts that will identify the article as belonging to both topics, with a specific weight for each.
</p>
<p>
A keyword, phrase, or term, is not a prisoner to a single concept at all. This is how algorithms in natural language understanding can understand how two topics are related (e.g. read about <a href="https://en.wikipedia.org/wiki/Topic_model" target="_blank" rel="nofollow noopener noreferrer"><em>topic modeling</em></a>). Each topic has a specific <strong>vocabulary</strong>, a list of words and phrases commonly used in its context, and some of these terms are present in different vocabularies.
</p>
<p>
Therefore, content depth and content breadth are not to be opposed. Content marketers should use both strategies in order to reach ultimate <strong>topical authority</strong> over their choice of subject matters.
</p>
<h2 id="depth-of-content--quality--frequency">
<a href="#depth-of-content--quality--frequency" aria-label="depth of content quality frequency permalink"></a>Depth of Content = Quality + Frequency
</h2>
<p>
Up until recently, long-form blog posts generally were <strong>evergreen articles</strong> that generated a constant stream of organic traffic for a website. This was a lead magnet generation strategy which worked well: hire a writer, include the right keywords, reach over a 5,000-word word count, and hit publish. Then, wait.
</p>
<p>
Nowadays, in-depth content requires more effort over time in order to pay off. Writing a big article, as good as it is, will not get your anywhere near the level of <a href="https://topicseed.com/blog/topical-seo" target="_blank" rel="nofollow noopener noreferrer">topical breadth</a>&nbsp;required by Google to rank you first. Instead, your content marketing plan should be about having:
</p>
<ul>
<li>a <strong>comprehensive pillar page</strong> covering a unique topic, and
</li>
<li>
<strong>narrow-focused children articles</strong> to dig deeper.
</li>
</ul>
<p>
Search engines also look at how often you publish about a specific topic, and when was the last time it was written about. Nobody likes a <a href="https://www.copypress.com/blog/avoiding-blog-graveyard/" target="_blank" rel="nofollow noopener noreferrer">graveyard blog</a>, it just makes the reader lose trust; as if the writer was not good enough, therefore had no traffic, before entirely giving up. Deep content requires a sustained effort on your part to always new find ways to write about a specific subject. Sure, it will be easy at first. But what about five years later? Well, you will still need to hit publish, all about the very same topics you already covered years ago.
</p>
<p>
Tools and platforms such as topicseed are here to <a href="https://topicseed.com/blog/how-to-find-new-blog-post-ideas" target="_blank" rel="nofollow noopener noreferrer">help you find new article ideas</a> pertaining to your core topics within a few clicks and a few minutes. The number of web pages, Wikipedia articles, and pieces of content, our machine-learning algorithms can analyze in seconds would take you months to digest. Our <em>topicgraph</em>&nbsp;finds closely related concepts in order for your domain to <strong>reach topical authority through content depth and content breadth</strong>.
</p>
</div>

400
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/topicseed-1/source.html vendored Normal file
View File

File diff suppressed because one or more lines are too long

1
plugins/af_readability/vendor/fivefilters/readability.php/test/test-pages/v8-blog/expected-images.json vendored Normal file
View File

@ -0,0 +1 @@
[]

Some files were not shown because too many files have changed in this diff Show More