Web Serving Stack

• Linux, Apache, MySQL, PHP
• Memcache, Gearman, StatsD, Vitess, Redis, Kafka, Varnish
• GCP, Terraform

API First!

Request Routing

https://etsy.com/awesome/123

.htaccess Apache rewrite rule

RewriteRule ^awesome/(\d+)$   /awesome.php?id=$1 [L,NC,QSA]

awesome.php

require 'bootstrap.php';
$request  = HTTP_Request::getInstance();
$response = HTTP_Response::getInstance();
$controller = new Awesome_Controller();
$controller->doCoolThings($request, $response);

Code/Awesome/Controller.php

class Awesome_Controller extends Controller_Base {
  public function doCoolThings($request, $response) {
    $id = $request->getGet('id', 0);
    if (!$id) {
      $response->redirect_error(Constants::ERROR_NOT_FOUND);
      return;
    }
    $thing = EtsyORM::getFinder('Thing')->findById($id);
    $stuff = Api::endpoint('AwesomeStuff', [$thing->id, 'max'=>10]);
    $this->renderViewTree(New Awesome_View($thing, $stuff));
  }
}

Awesome_View

class Awesome_View implements Neu_View {
    const TEMPLATE = "/templates/awesome/main.mustache";
    use Neu_Traits_DefaultView;

    public function __construct(AwesomeThing $thing, array $stuff) {
        $this->thing = $thing;
        $this->stuff = $stuff;
    }
    public function getCssFiles(): array {
        return [ '/awesome/main.scss' ];
    }
    public function getTemplateData(): array {
        return [ 'thing_id' => $this->thing->id,
                 'thing_name' => $this->thing->name,
                 'stuff' => $this->stuff ];
    }
}

templates/awesome/main.mustache

<div>
    <p>{{thing_name}} ({{thing_id}})</p>
    <ul>
    {{#stuff}}
        <li>{{id}} {{description}}</li>
    {{/stuff}}
    </ul>
</div>

Install with composer

$ composer require --dev phan/phan

Create .phan/config.php

return [
    'target_php_version' => '8.0',
    'directory_list' => [ 'src/' ],
    "exclude_analysis_directory_list" => [ 'vendor/' ],
];

$ ./vendor/bin/phan

Phan in Browser

Dependency Graph Plugin

Daemon mode

$ phan --daemonize-tcp-port default &
[1] 28610
Listening for Phan analysis requests at tcp://127.0.0.1:4846
Awaiting analysis requests for directory '/home/rasmus/phan_demo'

$ vi src/script.php

$ phan_client -l src/script.php
Phan error: TypeError: PhanTypeMismatchArgument: Argument 1 (union) is array{0:1} but \C::fn() takes int|string defined at src/script.php:8 in src/script.php on line 14
Phan error: TypeError: PhanTypeMismatchArgument: Argument 3 (shaped) is array{max:10} but \C::fn() takes array{mode:string,max:int} defined at src/script.php:8 in src/script.php on line 16

vim integration

Sample frequency in nanoseconds (or Hz)

$ phpspy -s 200000000 -- php -r 'sleep(1);' 
0 sleep <internal>:-1
1 <main> <internal>:-1

0 sleep <internal>:-1
1 <main> <internal>:-1

0 sleep <internal>:-1
1 <main> <internal>:-1

0 sleep <internal>:-1
1 <main> <internal>:-1

0 sleep <internal>:-1
1 <main> <internal>:-1

process_vm_readv: No such process

Attach to a running process

$ sudo phpspy -r -p $(pgrep -n php-fpm)

0 wp_installing /var/www/wordpress/wp-includes/load.php:944
1 wp_load_alloptions /var/www/wordpress/wp-includes/option.php:189
2 get_option /var/www/wordpress/wp-includes/option.php:90
3 create_initial_taxonomies /var/www/wordpress/wp-includes/taxonomy.php:43
4 WP_Hook::apply_filters /var/www/wordpress/wp-includes/class-wp-hook.php:286
5 WP_Hook::do_action /var/www/wordpress/wp-includes/class-wp-hook.php:310
6 do_action /var/www/wordpress/wp-includes/plugin.php:453
7 <main> /var/www/wordpress/wp-settings.php:450
8 <main> /var/www/wordpress/wp-config.php:89
9 <main> /var/www/wordpress/wp-load.php:37
10 <main> /var/www/wordpress/wp-blog-header.php:13
11 <main> /var/www/wordpress/index.php:17
# 1537119612.459615 /index.php p=1 /var/www/wordpress/index.php -

0 mysqli_query <internal>:-1
1 wpdb::_do_query /var/www/wordpress/wp-includes/wp-db.php:1924
2 wpdb::query /var/www/wordpress/wp-includes/wp-db.php:1813
3 wpdb::get_results /var/www/wordpress/wp-includes/wp-db.php:2488
4 _prime_comment_caches /var/www/wordpress/wp-includes/comment.php:2871
5 WP_Comment_Query::get_comments /var/www/wordpress/wp-includes/class-wp-comment-query.php:427
6 WP_Comment_Query::query /var/www/wordpress/wp-includes/class-wp-comment-query.php:346
7 get_comments /var/www/wordpress/wp-includes/comment.php:226
8 WP_Widget_Recent_Comments::widget /var/www/wordpress/wp-includes/widgets/class-wp-widget-recent-comments.php:99
9 WP_Widget::display_callback /var/www/wordpress/wp-includes/class-wp-widget.php:372
10 dynamic_sidebar /var/www/wordpress/wp-includes/widgets.php:743
11 <main> /var/www/wordpress/wp-content/themes/twentyfifteen/sidebar.php:41
12 load_template /var/www/wordpress/wp-includes/template.php:688
13 locate_template /var/www/wordpress/wp-includes/template.php:647
14 get_sidebar /var/www/wordpress/wp-includes/general-template.php:110
15 <main> /var/www/wordpress/wp-content/themes/twentyfifteen/header.php:49
16 load_template /var/www/wordpress/wp-includes/template.php:688
17 locate_template /var/www/wordpress/wp-includes/template.php:647
18 get_header /var/www/wordpress/wp-includes/general-template.php:41
19 <main> /var/www/wordpress/wp-content/themes/twentyfifteen/single.php:10
20 <main> /var/www/wordpress/wp-includes/template-loader.php:74
21 <main> /var/www/wordpress/wp-blog-header.php:19
22 <main> /var/www/wordpress/index.php:17
# 1537119612.459615 /index.php p=1 /var/www/wordpress/index.php -

Memory usage on stack frames

$ sudo phpspy -m php src/phan.php

0 Phan\Analysis::parseNodeInContext /home/rasmus/phan/src/Phan/Analysis.php:176
1 Phan\Analysis::parseNodeInContext /home/rasmus/phan/src/Phan/Analysis.php:176
2 Phan\Analysis::parseNodeInContext /home/rasmus/phan/src/Phan/Analysis.php:176
3 Phan\Analysis::parseNodeInContext /home/rasmus/phan/src/Phan/Analysis.php:176
4 Phan\Analysis::parseFile /home/rasmus/phan/src/Phan/Analysis.php:63
5 Phan\Phan::analyzeFileList /home/rasmus/phan/src/Phan/Phan.php:94
6 <main> /home/rasmus/phan/src/phan.php:1
# mem 119159776 123721960

0 ast\parse_code <internal>:-1
1 Phan\AST\Parser::parseCode /home/rasmus/phan/src/Phan/AST/Parser.php:42
2 Phan\Analysis::parseFile /home/rasmus/phan/src/Phan/Analysis.php:63
3 Phan\Phan::analyzeFileList /home/rasmus/phan/src/Phan/Phan.php:94
4 <main> /home/rasmus/phan/src/phan.php:1
# mem 82471616 123721960

Top-like output mode

Generate a flame graph

$ phpspy phan > /tmp/output
$ cat /tmp/output | stackcollapse-phpspy.pl | flamegraph.pl > flame.svg

Atomic

No performance hit

• No restarts
• No LB removal
• No thundering herd
• Cache reuse

Must be able to serve two versions of the site concurrently!

Requests that begin on DocumentRoot A must finish on A

Set the DocumentRoot to symlink target!

Easy with nginx

fastcgi_param DOCUMENT_ROOT $realpath_root

Apache

Avoid hardcoding full paths

Watch your include_path setting

incpath extension can resolve your include_path for you

Version all static assets

DB Schema changes need special care

At Etsy we use irc Slack

Channel: #push Topic: <prod> *joe frank|bob
devbot: Swapping symlinks. Your code is about to start taking production traffic
pushbot: joe frank : Your code is live. Time to watch graphs: http://etsy/abcd
Rasmus: .join
*** pushbot has changed the topic on #push to <prod> joe frank|bob Rasmus
frank: .good
*** pushbot has changed the topic on #push to <prod> *joe *frank|bob Rasmus
joe: .done
*** pushbot has changed the topic on #push to <prod> bob Rasmus
pushbot: bob Rasmus: You're up
bob: .in
*** pushbot has changed the topic on #push *bob Rasmus
Rasmus: .in
*** pushbot has changed the topic on #push *bob *Rasmus

pushbot commands

• .join    - join push queue
• .in        - code has been pushed
• .good - your stuff looks good
• .uhoh - your stuff looks bad
• .hold  - there is a problem, hold everything
• .nm     - never mind (leave queue)
• .done - push done

Channel: #push Topic: <princess> bob Rasmus
Jenkins: Starting build #36803 for job qa
Jenkins: Starting build #38784 for job princess
Jenkins: Project qa build #36803: SUCCESS in 6 min 19 sec: http://ci/job/qa/36803/
pushbot: bob Rasmus : qa tests have passed
devbot: [who_tried] Everyone in this push has run Try recently. w00t!
Jenkins: Project princess build #38784: SUCCESS in 1 min 10 sec: http://ci/job/princess/38784/
pushbot: bob Rasmus : princess tests have passed
bob: .good
Rasmus: .good
*** pushbot has changed the topic on #push to <princess> *bob *Rasmus
pushbot: bob Rasmus : everyone is ready, checking on Jenkins...
Jenkins: qa: last build: 36803 (9 min 5 sec ago): SUCCESS: http://ci/job/qa/36803/
Jenkins: princess: last build: 38784 (2 min 54 sec ago): SUCCESS: http://ci/job/princess/38784/

Deploy to Production:

• ssh to deploy host
• dsh to all targets
• rsync files

Channel: #push Topic: <prod> bob Rasmus
devbot: Swapping symlinks. Your code is about to start taking production traffic
pushbot: bob Rasmus : Your code is live. Time to watch graphs: http://etsy/et5cp
Jenkins: Starting build #39452 for job prod
pushbot: bob Rasmus : prod tests have passed
Jenkins: Project prod build #39452: SUCCESS in 30 sec: http://ci/job/prod/39452/
bob: .good
Rasmus: .good
*** pushbot has changed the topic on #push to <prod> *bob *Rasmus
pushbot: bob Rasmus : everyone is ready, checking on Jenkins...
Jenkins: prod: last build: 39452 (1 min 39 sec ago): SUCCESS: http://ci/job/prod/39452/
bob: .done
pushbot: clear
*** pushbot has changed the topic on #push to clear

Graph Everything!

• Statsd
• Grafana

Log Everything!

• Supergrep
• Logstash
• Elastic Search
• mtail
• Prometheus

• Commit to master
• Deploy from HEAD
• Branches?
• Branches are in code via feature flags

Blameless post-mortems

Named Arguments

htmlspecialchars($string, double_encode: false);

// instead of

htmlspecialchars($string, ENT_COMPAT | ENT_HTML401, 'UTF-8', false);

Constructor Property Promotion

class User {
    function __construct(
        public string $first_name,
        public string $last_name,
        private string $password = "",
        protected int $group = 1
    ) { }
}

$u = new User("Rasmus", "Lerdorf", group:2);
echo $u->first_name;  // Rasmus

Nullsafe Operator with short-circuiting

Similar to ?. operator in Javascript, C# and Swift

$country = $session?->user?->getAddress(geoip())?->country;

// instead of

if ($session !== null) {
    $user = $session->user;

    if ($user !== null) {
        $address = $user->getAddress(geoip());

        if ($address !== null) {
            $country = $address->country;
        }
    }
}

Match Expression

$statement = match ($this->lexer->lookahead['type']) {
    Lexer::T_SELECT => $this->SelectStatement(),
    Lexer::T_UPDATE => $this->UpdateStatement(),
    Lexer::T_DELETE => $this->DeleteStatement(),
    default => $this->syntaxError('SELECT, UPDATE or DELETE'),
};
// Throws UnhandledMatchError on no match and no default expr

// instead of
switch ($this->lexer->lookahead['type']) {
    case Lexer::T_SELECT:
        $statement = $this->SelectStatement();
        break;
    case Lexer::T_UPDATE:
        $statement = $this->UpdateStatement();
        break;
    case Lexer::T_DELETE:
        $statement = $this->DeleteStatement();
        break;
    default:
        $this->syntaxError('SELECT, UPDATE or DELETE');
        break;
}

Union Types

class Store {
    private static $data = [];

    /**
     * @param int|string $key
     * @param int|float|string $val
     */
    static function add($key, $val): void {
        if(!(is_int($key) || is_string($key))) {
            throw new TypeError("Key must be an int or a string");
        }
        if(!(is_int($val) || is_float($val) || is_string($val))) {
            throw new TypeError("Value must be an int, float or a string");
        }
        self::$data[$key] = $val;
    }
    /**
     * @param int|string $key
     * @return int|float|string
     */
    static function get($key) {
        return self::$data[$key];
    }
}

Union Types

<?php
class Store {
    private static $data = [];

    static function add(int|string $key, int|float|string $val): void {
        self::$data[$key] = $val;
    }
    static function get(int|string $key): int|float|string {
        return self::$data[$key];
    }
}
Store::add('player2', [1,2,3]);
// TypeError: Store::add(): Argument #2 ($val) must be of
//                          type string|int|float, array given

weakMap

Map objects to arbitrary values without preventing GC

class Endpoint {
    function __construct(public string $url, ?callable $dfunc=null, array $opts = []) {
        $this->context = stream_context_create($opts);
        $this->dfunc = $dfunc ? $dfunc : fn($x)=>$x;
    }
}

class Api {
    static public ?weakMap $cache = null;
    static public function fetch(Endpoint $ep): string|object {
        if(!self::$cache) self::$cache = new weakMap;
        return self::$cache[$ep] ??=
               $ep->dfunc->call($ep, file_get_contents($ep->url, context:$ep->context));
    }
}

$xkcd = new Endpoint("http://xkcd.com/info.0.json", fn($x)=>json_decode($x, associative:false));
$joke = new Endpoint("https://icanhazdadjoke.com/", opts:['http'=>['header'=>"Accept:text/plain"]]);
echo '<img src="'.Api::fetch($xkcd)->img.'" alt="'.Api::fetch($xkcd)->alt.'">'."\n";
echo Api::fetch($joke) . "\n";
echo Api::$cache->count() . "\n"; // 2
unset($xkcd);
echo Api::$cache->count() . "\n"; // 1
echo Api::fetch($joke) . "\n";    // Same bad joke
$joke = new Endpoint("https://icanhazdadjoke.com/", opts:['http'=>['header'=>"Accept:text/plain"]]);
echo Api::fetch($joke) . "\n";    // New bad joke
echo Api::$cache->count() . "\n"; // 2?
gc_collect_cycles();              // Force gc
echo Api::$cache->count() . "\n"; // 1

Attributes

use Doctrine\ORM\Attributes as ORM;
use Symfony\Component\Validator\Constraints as Assert;

#[ORM\Entity]
class User {
    #[ORM\Id, ORM\Column("integer"), ORM\GeneratedValue]
    private $id;

    #[ORM\Column("string", ORM\Column::UNIQUE)]
    #[Assert\Email(array("message" => "The email '{{ value }}' is not a valid email."))]
    private $email;

    #[Assert\Range(["min" => 120, "max" => 180, "minMessage" => "You must be at least {{ limit }}cm tall to enter"])]
    #[ORM\Column(ORM\Column::T_INTEGER)]
    protected $height;

    #[ORM\ManyToMany(Phonenumber::class)]
    #[
       ORM\JoinTable("users_phonenumbers"),
       ORM\JoinColumn("user_id", "id"),
       ORM\InverseJoinColumn("phonenumber_id", "id", ORM\JoinColumn::UNIQUE)
     ]
    private $phonenumbers;
}

Create more value than you capture. -Tim O'Reilly

Work on things that matter (to you)