<?xml version="1.0" encoding="US-ASCII"?>
<!-- This template is for creating an Internet Draft using xml2rfc,
     which is available here: http://xml.resource.org. -->
<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [
<!-- One method to get references from the online citation libraries.
     There has to be one entity for each item to be referenced. 
     An alternate method (rfc include) is described in the references. -->
<!ENTITY RFC4884 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.4884.xml">
<!ENTITY RFC4950 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.4950.xml">
<!ENTITY RFC5837 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.5837.xml">
<!ENTITY RFC7665 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7665.xml">
<!ENTITY RFC7872 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7872.xml">
<!ENTITY RFC2119 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.2119.xml">
<!ENTITY RFC6830 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.6830.xml">
<!ENTITY RFC7112 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7112.xml">
<!ENTITY RFC6833 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.6833.xml">
<!ENTITY RFC7276 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7276.xml">
<!ENTITY RFC2460 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.2460.xml">
<!ENTITY RFC7799 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.7799.xml">
<!ENTITY RFC791 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.0791.xml">
<!ENTITY RFC6564 SYSTEM "http://www.rfc-editor.org/refs/bibxml/reference.RFC.6564.xml">
<!ENTITY I-D.brockners-inband-oam-data SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.brockners-inband-oam-data.xml">
<!ENTITY I-D.brockners-inband-oam-requirements SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.brockners-inband-oam-requirements.xml">
<!ENTITY I-D.brockners-inband-oam-transport SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.brockners-inband-oam-transport.xml">
<!ENTITY I-D.brockners-proof-of-transit SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.brockners-proof-of-transit.xml">
<!ENTITY I-D.ietf-spring-segment-routing SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.ietf-spring-segment-routing.xml">
<!ENTITY I-D.previdi-isis-segment-routing-extensions SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.previdi-isis-segment-routing-extensions.xml">
<!ENTITY I-D.ietf-ippm-6man-pdm-option SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.ietf-ippm-6man-pdm-option.xml">
<!ENTITY I-D.brockners-lisp-sr SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.brockners-lisp-sr.xml">
<!ENTITY I-D.hildebrand-spud-prototype SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.hildebrand-spud-prototype.xml">
<!ENTITY I-D.ietf-sfc-nsh SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.ietf-sfc-nsh.xml">
<!ENTITY I-D.ietf-6man-segment-routing-header SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.ietf-6man-segment-routing-header.xml">
<!ENTITY I-D.ietf-nvo3-vxlan-gpe SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.ietf-nvo3-vxlan-gpe.xml">
<!ENTITY I-D.lapukhov-dataplane-probe SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.lapukhov-dataplane-probe.xml">
<!ENTITY I-D.SPUD SYSTEM "http://www.rfc-editor.org/refs/bibxml3/reference.I-D.hildebrand-spud-prototype.xml">
<!ENTITY AFI SYSTEM "http://www.iana.org/assignments/address-family-numbers/address-family-numbers.xml">
]>
<?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?>
<!-- used by XSLT processors -->
<!-- For a complete list and description of processing instructions (PIs), 
     please see http://xml.resource.org/authoring/README.html. -->
<!-- Below are generally applicable Processing Instructions (PIs) that most I-Ds might want to use.
     (Here they are set differently than their defaults in xml2rfc v1.32) -->
<?rfc strict="yes" ?>
<!-- give errors regarding ID-nits and DTD validation -->
<!-- control the table of contents (ToC) -->
<?rfc toc="yes"?>
<!-- generate a ToC -->
<?rfc tocdepth="4"?>
<!-- the number of levels of subsections in ToC. default: 3 -->
<!-- control references -->
<?rfc symrefs="yes"?>
<!-- use symbolic references tags, i.e, [RFC2119] instead of [1] -->
<?rfc sortrefs="yes" ?>
<!-- sort the reference entries alphabetically -->
<!-- control vertical white space 
     (using these PIs as follows is recommended by the RFC Editor) -->
<?rfc compact="yes" ?>
<!-- do not start each main section on a new page -->
<?rfc subcompact="no" ?>
<!-- keep one blank line between list items -->
<!-- end of list of popular I-D processing instructions -->
<rfc category="info" docName="draft-brockners-inband-oam-requirements-02"
     ipr="trust200902">
  <!-- ipr="full3978"-->

  <!-- category values: std, bcp, info, exp, and historic
     ipr values: full3667, noModification3667, noDerivatives3667
     you can add the attributes updates="NNNN" and obsoletes="NNNN" 
     they will automatically be output with "(if approved)" -->

  <!-- ***** FRONT MATTER ***** -->

  <front>
    <!-- The abbreviated title is used in the page header - it is only necessary if the 
         full title is longer than 39 characters -->

    <title abbrev="In-situ OAM Requirements">Requirements for In-situ
    OAM</title>

    <!-- add 'role="editor"' below for the editors if appropriate -->

    <!-- Another author who claims to be an editor -->

    <author fullname="Frank Brockners" initials="F." surname="Brockners">
      <organization abbrev="Cisco">Cisco Systems, Inc.</organization>

      <address>
        <postal>
          <street>Hansaallee 249, 3rd Floor</street>

          <!-- Reorder these if your country does things differently -->

          <city>DUESSELDORF</city>

          <region>NORDRHEIN-WESTFALEN</region>

          <code>40549</code>

          <country>Germany</country>
        </postal>

        <email>fbrockne@cisco.com</email>

        <!-- uri and facsimile elements may also be added -->
      </address>
    </author>

    <author fullname="Shwetha Bhandari" initials="S." surname="Bhandari">
      <organization abbrev="Cisco">Cisco Systems, Inc.</organization>

      <address>
        <postal>
          <street>Cessna Business Park, Sarjapura Marathalli Outer Ring
          Road</street>

          <city>Bangalore, KARNATAKA 560 087</city>

          <country>India</country>
        </postal>

        <email>shwethab@cisco.com</email>
      </address>
    </author>

    <author fullname="Sashank Dara" initials="S." surname="Dara">
      <organization abbrev="Cisco">Cisco Systems, Inc.</organization>

      <address>
        <postal>
          <street>Cessna Business Park, Sarjapura Marathalli Outer Ring
          Road</street>

          <city>Bangalore, KARNATAKA 560 087</city>

          <country>India</country>
        </postal>

        <email>sadara@cisco.com</email>
      </address>
    </author>

    <author fullname="Carlos Pignataro" initials="C." surname="Pignataro">
      <organization abbrev="Cisco">Cisco Systems, Inc.</organization>

      <address>
        <postal>
          <street>7200-11 Kit Creek Road</street>

          <city>Research Triangle Park</city>

          <region>NC</region>

          <code>27709</code>

          <country>United States</country>
        </postal>

        <email>cpignata@cisco.com</email>
      </address>
    </author>

    <author fullname="Hannes Gredler" initials="H." surname="Gredler">
      <organization>RtBrick Inc.</organization>

      <address>
        <email>hannes@rtbrick.com</email>
      </address>
    </author>

    <author fullname="John Leddy" initials="J." surname="Leddy">
      <organization abbrev="Comcast">Comcast</organization>

      <address>
        <email>John_Leddy@cable.comcast.com</email>
      </address>
    </author>

    <author fullname="Stephen Youell" initials="S." surname="Youell">
      <organization abbrev="JMPC">JP Morgan Chase</organization>

      <address>
        <postal>
          <street>25 Bank Street</street>

          <city>London</city>

          <code>E14 5JP</code>

          <country>United Kingdom</country>
        </postal>

        <email>stephen.youell@jpmorgan.com</email>
      </address>
    </author>

    <author fullname="David Mozes" initials="D." surname="Mozes">
      <organization>Mellanox Technologies Ltd.</organization>

      <address>
        <postal>
          <street/>

          <city/>

          <region/>

          <code/>

          <country/>
        </postal>

        <phone/>

        <facsimile/>

        <email>davidm@mellanox.com</email>

        <uri/>
      </address>
    </author>

    <author fullname="Tal Mizrahi" initials="T." surname="Mizrahi">
      <organization>Marvell</organization>

      <address>
        <postal>
          <street>6 Hamada St.</street>

          <city>Yokneam</city>

          <region/>

          <code>20692</code>

          <country>Israel</country>
        </postal>

        <phone/>

        <facsimile/>

        <email>talmi@marvell.com</email>

        <uri/>
      </address>
    </author>

    <author fullname="Petr Lapukhov" initials="P." surname="Lapukhov">
      <organization>Facebook</organization>

      <address>
        <postal>
          <street>1 Hacker Way</street>

          <city>Menlo Park</city>

          <region>CA</region>

          <code>94025</code>

          <country>USA</country>
        </postal>

        <phone/>

        <facsimile/>

        <email/>

        <uri>petr@fb.com</uri>
      </address>
    </author>

    <author fullname="Remy Chang" initials="R." surname="Chang">
      <organization>Barefoot Networks</organization>

      <address>
        <postal>
          <street/>

          <city/>

          <region/>

          <code/>

          <country/>
        </postal>

        <phone/>

        <facsimile/>

        <email>remy@barefootnetworks.com</email>

        <uri/>
      </address>
    </author>

    <date day="30" month="October" year="2016"/>

    <!-- If the month and year are both specified and are the current ones, xml2rfc will fill 
         in the current day for you. If only the current year is specified, xml2rfc will fill 
	 in the current day and month for you. If the year is not the current one, it is 
	 necessary to specify at least a month (xml2rfc assumes day="1" if not specified for the 
	 purpose of calculating the expiry date).  With drafts it is normally sufficient to 
	 specify just the year. -->

    <!-- Meta-data Declarations -->

    <area>ops</area>

    <!-- WG name at the upperleft corner of the doc,
         IETF is fine for individual submissions.  
	 If this element is not present, the default is "Network Working Group",
         which is used by the RFC Editor as a nod to the history of the IETF. -->

    <keyword>Telemetry, Tracing,</keyword>

    <!-- Keywords will be incorporated into HTML output
         files in a meta tag but they have no effect on text or nroff
         output. If you submit your draft to the RFC Editor, the
         keywords will be used for the search engine. -->

    <abstract>
      <t>This document discusses the motivation and requirements for including
      specific operational and telemetry information into data packets while
      the data packet traverses a path between two points in the network. This
      method is referred to as "in-situ" Operations, Administration, and
      Maintenance (OAM), given that the OAM information is carried with the
      data packets as opposed to in "out-of-band" packets dedicated to OAM. In
      situ OAM complements other OAM mechanisms which use dedicated probe
      packets to convey OAM information.</t>
    </abstract>
  </front>

  <middle>
    <section title="Introduction" toc="default">
      <t>This document discusses requirements for "in-situ" Operations,
      Administration, and Maintenance (OAM) mechanisms. In this context,
      &ldquo;in-situ OAM&rdquo; refers to the concept of directly encoding
      telemetry information within the data packet as it traverses the network
      or telemetry domain. Mechanisms which add tracing or other types of
      telemetry information to the regular data traffic, sometimes also
      referred to as "in-band" OAM can complement active, probe-based
      mechanisms such as ping or traceroute, which are sometimes considered as
      "out-of-band", because the messages are transported independently from
      regular data traffic. In terms of "active" or "passive" OAM, "in-situ"
      OAM can be considered a hybrid OAM type. While no extra packets are
      sent, in-situ OAM adds information to the packets therefore cannot be
      considered passive. In terms of the classification given in <xref
      target="RFC7799"/> in-situ OAM could be portrayed as "hybrid OAM, type
      1". "In-situ" mechanisms do not require extra packets to be sent and
      hence don't change the packet traffic mix within the network. Traceroute
      and ping for example use ICMP messages: New packets are injected to get
      tracing information. Those add to the number of messages in a network,
      which already might be highly loaded or suffering performance issues for
      a particular path or traffic type.</t>

      <t>A number of in-situ as well as in-band OAM mechanisms have been
      discussed, such as the INT spec for the P4 programming language <xref
      target="P4"/> or the SPUD prototype <xref
      target="I-D.hildebrand-spud-prototype"/>. The SPUD prototype uses a
      similar logic that allows network devices on the path between endpoints
      to participate explicitly in the tube outside the end-to-end context.
      Even the IPv4 route-record option defined in <xref target="RFC0791"/>
      can be considered an in-situ OAM mechanism. Per what was already stated,
      in-situ OAM complements "out-of-band" mechanisms such as ping or
      traceroute, or more recent active probing mechanisms, as described in
      <xref target="I-D.lapukhov-dataplane-probe"/>. In-situ OAM mechanisms
      can be leveraged where current out-of-band mechanisms do not apply or do
      not offer the desired characteristics or requirements, such as proving
      that a certain set of traffic takes a pre-defined path, strict
      congruency between overlay and underlay transports is in place, checking
      service level agreements for the live data traffic, detailed statistics
      or verification of path selections within a domain, or scenarios where
      probe traffic is potentially handled differently from regular data
      traffic by the network devices. <xref target="RFC7276"/> presents an
      overview of OAM tools.</t>

      <t>Compared to probably the most basic example of "in-situ OAM" which is
      IPv4 route recording <xref target="RFC0791"/>, an in-situ OAM approach
      has the following capabilities:<list style="letters">
          <t>A flexible data format to allow different types of information to
          be captured as part of an in-situ OAM operation, including but not
          limited to path tracing information, operational and telemetry
          information such as timestamps, sequence numbers, or even generic
          data such as queue size, geo-location of the node that forwarded the
          packet, etc.</t>

          <t>A data format to express node as well as link identifiers to
          record the path a packet takes with a fixed amount of added
          data.</t>

          <t>The ability to determine whether any nodes were skipped while
          recording in-situ OAM information (i.e., in-situ OAM is not
          supported or not enabled on those nodes).</t>

          <t>The ability to actively process information in the packet, for
          example to prove in a cryptographically secure way that a packet
          really took a pre-defined path using some traffic steering method
          such as service chaining or traffic engineering.</t>

          <t>The ability to include OAM data beyond simple path information,
          such as timestamps or even generic data of a particular use
          case.</t>

          <t>The ability to carry in-situ OAM data in various different
          transport protocols.</t>
        </list></t>
    </section>

    <section anchor="Conventions" title="Conventions">
      <t>The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
      "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this
      document are to be interpreted as described in <xref
      target="RFC2119"/>.</t>

      <t>Abbreviations used in this document:</t>

      <t><list hangIndent="11" style="hanging">
          <t hangText="ECMP:">Equal Cost Multi-Path</t>

      <t hangText="LISP:">Locator/ID Separation Protocol</t>

          <t hangText="MTU:">Maximum Transmit Unit</t>

          <t hangText="NSH:">Network Service Header</t>

          <t hangText="NFV:">Network Function Virtualization</t>

          <t hangText="OAM:">Operations, Administration, and Maintenance</t>

          <t hangText="PMTU:">Path MTU</t>

          <t hangText="SFC:">Service Function Chain</t>

          <t hangText="SLA:">Service Level Agreement</t>

          <t hangText="SR:">Segment Routing</t>

          <!--

          <t hangText="SID:">Segment Identifier</t>


          <t hangText="VXLAN-GPE:">Virtual eXtensible Local Area Network,
             Generic Protocol Extension</t>

-->
        </list></t>

      <t>This document defines in-situ Operations, Administration, and
      Maintenance (in-situ OAM), as the subset in which OAM information is
      carried along with data packets. This is as opposed to "out-of-band
      OAM", where specific packets are dedicated to carrying OAM
      information.</t>
    </section>

    <section title="Motivation for in-situ OAM">
      <t>In several scenarios it is beneficial to make information about the
      path a packet took through the network or through a network device as
      well as associated telemetry information available to the operator. This
      includes not only tasks like debugging, troubleshooting, as well as
      network planning and network optimization but also policy or service
      level agreement compliance checks. This section discusses the motivation
      to introduce new methods for enhanced in-situ network diagnostics.</t>

      <section title="Path Congruency Issues with Dedicated OAM Packets">
        <t>Packet scheduling algorithms, especially for balancing traffic
        across equal cost paths or links, often leverage information contained
        within the packet, such as protocol number, IP-address or MAC-address.
        Probe packets would thus either need to be sent from the exact same
        endpoints with the exact same parameters, or probe packets would need
        to be artificially constructed as "fake" packets and inserted along
        the path. Both approaches are often not feasible from an operational
        perspective, be it that access to the end-system is not feasible, or
        that the diversity of parameters and associated probe packets to be
        created is simply too large. An in-situ mechanism is an alternative in
        those cases.</t>

        <t>In-situ mechanisms are not impacted by differences in the handling
        of probe traffic compared to other data packets, where probe traffic
        is handled differently (and potentially forwarded differently) by a
        router than regular data traffic. This obviously assumes that the
        addition of in-situ information does not change the forwarding
        behavior of the packet. Note that in certain implementations, the
        addition information to a transport protocol changes the forwarding
        behavior. IPv6 extension header processing is one example. Some
        implementations process IPv6 packets with extension headers in the
        "slow" path of a router, as opposed to the "fast" path.</t>
      </section>

      <section title="Results Sent to a System Other Than the Sender">
        <t>Traditional ping and traceroute tools return the OAM results to the
        sender of the probe. Even when the ICMP messages that are used with
        these tools are enhanced, and additional telemetry is collected (e.g.,
        ICMP Multi-Part <xref target="RFC4884"/> supporting MPLS information
        <xref target="RFC4950"/>, Interface and Next-Hop Identification <xref
        target="RFC5837"/>, etc.), it would be advantageous to separate the
        sending of an OAM probe from the receiving of the telemetry data. In
        this context, it is helpful to eliminate the requirement that there be
        a working bidirectional path.</t>
      </section>

      <section title="Overlay and Underlay Correlation">
        <t>Several network deployments leverage tunneling mechanisms to create
        overlay or service-layer networks. Examples include VXLAN-GPE, GRE, or
        LISP. One often observed attribute of overlay networks is that they do
        not offer the user of the overlay any insight into the underlay
        network. This means that the path that a particular tunneled packet
        takes, nor other operational details such as the per-hop delay/jitter
        in the underlay are visible to the user of the overlay network, giving
        rise to diagnosis and debugging challenges in case of connectivity or
        performance issues. The scope of OAM tools like ping or traceroute is
        limited to either the overlay or the underlay which means that the
        user of the overlay has typically no access to OAM in the underlay,
        unless specific operational procedures are put in place. With in-situ
        OAM the operator of the underlay can offer details of the connectivity
        in the underlay to the user of the overlay. This could include the
        ability to find out which underlay elements are shared by overlays and
        ability to know which overlays are mapped to the same underlay
        elements. Deployment dependent underlay transit nodes can be
        configured to update OAM information in the overlay transport
        encapsulation. The operator of the egress tunnel router could choose
        to share the recorded information about the path with the user of the
        overlay.</t>

        <t>Coupled with mechanisms such as Segment Routing (SR) <xref
        target="I-D.ietf-spring-segment-routing"/>, overlay network and
        underlay network can be more tightly coupled: The user of the overlay
        has detailed diagnostic information available in case of failure
        conditions. The user of the overlay can also use the path recording
        information as input to traffic steering or traffic engineering
        mechanisms, to for example achieve path symmetry for the traffic
        between two endpoints. <xref target="I-D.brockners-lisp-sr"/> is an
        example for how these methods can be applied to LISP.</t>
      </section>

      <section title="SLA Verification">
        <t>In-situ OAM can help users of an overlay-service to verify that
        negotiated SLAs for the real traffic are met by the underlay network
        provider. Different from solutions which rely on active probes to test
        an SLA, in-situ OAM based mechanisms avoid wrong interpretations and
        "cheating", which can happen if the probe traffic that is used to
        perform SLA-check is prioritized by the network provider of the
        underlay. In active/standby deployments in-situ OAM would only allow
        for SLA verification of the active path.</t>
      </section>

      <section title="Analytics and Diagnostics">
        <t>Network planners and operators benefit from knowledge of the actual
        traffic distribution in the network. When deriving an overall network
        connectivity traffic matrix one typically needs to correlate data
        gathered from each individual device in the network. If the path of a
        packet is recorded while the packet is forwarded, the entire path that
        a packet took through the network is available to the egress system.
        This obviates the need to retrieve individual traffic statistics from
        every device in the network and correlate those statistics, or employ
        other mechanisms such as leveraging traffic engineering with
        null-bandwidth tunnels just to retrieve the appropriate statistics to
        generate the traffic matrix.</t>

        <t>In addition, with individual path tracing, information is available
        at packet level granularity, rather than only at aggregate level - as
        is usually the case with IPFIX-style methods which employ flow-filters
        at the network elements. Data-center networks which use equal-cost
        multipath (ECMP) forwarding are one example where detailed statistics
        on flow distribution in the network are highly desired. If a network
        supports ECMP, one can create detailed statistics for the different
        paths packets take through the network at the egress system, without a
        need to correlate/aggregate statistics from every router in the
        system. Transit devices are off-loaded from the task of gathering
        packet statistics.</t>

        <t>In high-speed networks one can leverage and benefit from
        packet-accurate measurements with for example hardware-accurate
        timestamping (i.e., nanosecond-level verification) to support optimized
        packet scheduling and queuing mechanisms.</t>
      </section>

      <section title="Frame Replication/Elimination Decision for Bi-casting/Active-active Networks">
        <t>Bandwidth- and power-constrained, time-sensitive, or
        loss-intolerant networks (e.g., networks for industry
        automation/control, health care) require efficient OAM methods to
        decide when to replicate packets to a secondary path in order to keep
        the loss/error-rate for the receiver at a tolerable level - and also
        when to stop replication and eliminate the redundant flow. Many Internet of Things (IoT)
        networks are time sensitive and cannot leverage automatic
        retransmission requests (ARQ) to cope with transmission errors or lost
        packets. Transmitting the data over multiple disparate paths (often
        called bi-casting or live-live) is a method used to reduce the error
        rate observed by the receiver. Time sensitive networks (TSN) receive a
        lot of attention from the manufacturing industry as shown by a various
        standardization activities and industry forums being formed (see e.g.,
        IETF 6TiSCH, IEEE P802.1CB, AVnu).</t>
      </section>

      <section anchor="usecase_proof_of_transit" title="Proof of Transit">
        <t>Several deployments use traffic engineering, policy routing,
        segment routing or Service Function Chaining (SFC) <xref
        target="RFC7665"/> to steer packets through a specific set of nodes.
        In certain cases regulatory obligations or a compliance policy require
        to prove that all packets that are supposed to follow a specific path
        are indeed being forwarded across the exact set of nodes specified. If
        a packet flow is supposed to go through a series of service functions
        or network nodes, it has to be proven that all packets of the flow
        actually went through the service chain or collection of nodes
        specified by the policy. In case the packets of a flow weren't
        appropriately processed, a verification device would be required to
        identify the policy violation and take corresponding actions (e.g.,
        drop or redirect the packet, send an alert etc.) corresponding to the
        policy. In today's deployments, the proof that a packet traversed a
        particular service chain is typically delivered in an indirect way:
        Service appliances and network forwarding are in different trust
        domains. Physical hand-off-points are defined between these trust
        domains (i.e., physical interfaces). Or in other terms, in the
        "network forwarding domain" things are wired up in a way that traffic
        is delivered to the ingress interface of a service appliance and
        received back from an egress interface of a service appliance. This
        "wiring" is verified and trusted. The evolution to Network Function
        Virtualization (NFV) and modern service chaining concepts (using
        technologies such as 
      Locator/ID Separation Protocol (LISP), Network Service Header (NSH),
Segment Routing (SR), etc.) blurs the line
        between the different trust domains, because the hand-off-points are
        no longer clearly defined physical interfaces, but are virtual
        interfaces. Because of that very reason, networks operators require
        that different trust layers not to be mixed in the same device. For an
        NFV scenario a different proof is required. Offering a proof that a
        packet traversed a specific set of service functions would allow
        network operators to move away from the above described indirect
        methods of proving that a service chain is in place for a particular
        application.</t>

        <t>Deployed service chains without the presence of a "proof of
        transit" mechanism are typically operated as fail-open system: The
        packets that arrive at the end of a service chain are processed.
        Adding "proof of transit" capabilities to a service chain allows an
        operator to turn a fail-open system into a fail-close system, i.e.
        packets that did not properly traverse the service chain can be
        blocked.</t>

        <t>A solution approach could be based on OAM data which is added to
        every packet for achieving Proof Of Transit (POT).The OAM data is
        updated at every hop and is used to verify whether a packet traversed
        all required nodes. When the verifier receives each packet, it can
        validate whether the packet traversed the service chain correctly. The
        detailed mechanisms used for path verification along with the
        procedures applied to the OAM data carried in the packet for path
        verification are beyond the scope of this document. Details are
        addressed in <xref target="I-D.brockners-proof-of-transit"/>. In this
        document the term "proof" refers to a discrete set of bits that
        represents an integer or string carried as OAM data. The OAM data is
        used to verify whether a packet traversed the nodes it is supposed to
        traverse.</t>
      </section>

      <section title="Use Cases">
        <t>In-situ OAM could be leveraged for several use cases,
        including:<list style="symbols">
            <t>Traffic Matrix: Derive the network traffic matrix: Traffic for
            a given time interval between any two edge nodes of a given
            domain. Could be performed for all traffic or on a per Quality of Service (QoS) class.</t>

            <t>Flow Debugging: Discover which path(s) a particular set of
            traffic (identified by an n-tuple) takes in the network. Such a
            procedure is particularly useful in case traffic is balanced
            across multiple paths, like with link aggregation (LACP) or equal
            cost multi-pathing (ECMP).</t>

            <t>Loss Statistics per Path: Retrieve loss statistics per flow and
            path in the network.</t>

            <t>Path Heat Maps: Discover highly utilized links in the
            network.</t>

            <t>Trend Analysis on Traffic Patterns: Analyze if (and if so how)
            the forwarding path for a specific set of traffic changes over
            time (can give hints to routing issues, unstable links etc.)</t>

            <t>Network Delay Distribution: Show delay distribution across
            network by node or links. If enabled per application or for a
            specific flow then display the path taken along with the delay
            incurred at every hop.</t>

            <t>SLA Verification: Verify that a negotiated service level
            agreement (SLA), e.g., for packet drop rates or delay/jitter is
            conformed to by the actual traffic.</t>

            <t>Low-power Networks: Include application level OAM information
            (e.g., battery charge level, cache or buffer fill level) into data
            traffic to avoid sending extra OAM traffic which incur an extra
            cost on the devices. Using the battery charge level as example,
            one could avoid sending extra OAM packets just to communicate
            battery health, and as such would save battery on sensors.</t>

            <t>Path Verification or Service Function Path Verification: Proof
            and verification of packets traversing check points in the
            network, where check points can be nodes in the network or service
            functions.</t>

            <t>Geo-location Policy: Network policy implemented based on which
            path packets took. Example: Only if packets originated and stayed
            within the trading-floor department, access to specific
            applications or servers is granted.</t>

            <t>Device-level Troubleshooting and Optimization: In many cases,
            network operators could benefit from information specific to a
            single device. A non-exhaustive list of useful information
            includes: queue-depths, buffer utilization (either shared or
            per-port), packet latency measured from a known starting point,
            packet latency introduced by a single device, and resource
            utilization (CPU, memory, link bandwidth) of a given device or
            link. In some cases, this information changes over per-packet
            timescales (i.e., nanoseconds) and as such it is extremely
            challenging to collect and report this info in an accurate and
            scalable manner. By encoding the information from the forwarding
            element directly within a data packet (i.e., within the
            &lsquo;fast-path&rsquo;) this information can be added to some or
            all data packets and then collected and analyzed by human or
            machine tools. This type of information is particularly valuable
            for troubleshooting low-level device errors as well as providing a
            knowledge feedback loop for network and device optimization.</t>

            <t>Custom Network Probing: Active network probing and in-situ OAM
            can be combined for customized and efficient network probing. This
            could for example be a customized traceroute.</t>
          </list></t>
      </section>
    </section>

    <section title="Considerations for In-situ OAM">
      <t>The implementation of an in-situ OAM mechanism needs to take several
      considerations into account, including administrative boundaries, how
      information is recorded, Maximum Transfer Unit (MTU), Path MTU Discovery
(PMTUD)
      and packet size, etc.</t>

      <section anchor="type_of_iOAM_data"
               title="Type of Information to Be Recorded">
        <t>The information gathered for in-situ OAM can be categorized into
        three main categories: Information with a per-hop scope, such as path
        tracing; information which applies to a specific set of hops, such as
        path or service chain verification; information which only applies to
        the edges of a domain, such as sequence numbers. Note that a single
        network device could comprise several in-situ OAM hops, for example in
        case one wants to trace the path of a packet through that device.</t>

        <t><list style="symbols">
            <t>"edge to edge": Information that needs to be shared between
            network edges (the "edge" of a network could either be a host or a
            domain edge device): Edge to edge data e.g., packet and octet
            count of data entering a well-defined domain and leaving it is
            helpful in building traffic matrix, sequence number (also called
            &ldquo;path packet counters&rdquo;) is useful for the flow to
            detect packet loss.</t>

            <t>"selected hops": Information that applies to a specific set of
            nodes only. In case of path verification, only the nodes which are
            "check points" are required to interpret and update the
            information in the packet.</t>

            <t>"per hop": Information that is gathered at every hop along the
            path a packet traverses within an administrative domain:<list
                style="symbols">
                <t>Hop by Hop information e.g., Nodes visited for path
                tracing, Timestamps at each hop to find delays along the
                path</t>

                <t>Stats collection at each hop to optimize communication in
                resource constrained networks e.g., battery, CPU, memory
                status of each node piggy backed in a data packet is useful in
                low power lossy networks where network nodes are mostly asleep
                and communication is expensive</t>
              </list></t>
          </list></t>
      </section>

      <section title="MTU and Packet Size">
        <t>The recorded data at every hop might lead to packet size exceeding
        the Maximum Transmit Unit (MTU). A detailed discussion of the
        implications of oversized IPv6 header chains is found in <xref
        target="RFC7112"/>. The Path MTU restricts the amount of data that can
        be recorded for purpose of OAM within a data packet.</t>

        <t>If in-situ OAM data is inserted at the edge of the domain (e.g., by
        intermediate routers) then the MTU on all interfaces with the domain
        (MTU_INT) MUST be &gt;= the maximum MTU on any "external" facing
        interfaces (MTU_EXT) and the total size of in-situ OAM data to be
        recorded MUST be &lt;= (MTU_INT - MTU_EXT).</t>

        <t>In-situ OAM comprises two approaches to insert OAM data-records in
        the packets:</t>

        <t><list style="symbols">
            <t>Pre-allocated: In this case, the encapsulating node inserts
            empty data records into the packet to cover the entire domain. The
            data records will be incrementally updated/filled as the packet
            progresses through the network. With pre-allocation the packet
            size is only changed at the encapsulating node and is kept
            constant throughout the domain. The pre-allocated approach is
            beneficial for software data-plane implementations where
            allocating the required space only once and index into the array
            to populate the data during transit avoids copy operations at
            every hop.</t>

            <t>Incremental: Every node that desires to include in-situ OAM
            information extends the packet as needed. The incremental approach
            is beneficial for hardware data-plane implementations as it
            eliminates the need for the transit nodes to read the full array
            and lookup the pointer in the option prior to updating the data
            record contents.</t>
          </list>The "incremental" or the "pre-allocated" approaches could
        even be combined in the same deployment - in which case two in-situ
        OAM headers would be present in the packet: One for the incremental
        approach and one for the pre-allocated approach. In such a case one
        would expect that nodes with a hardware data-plane would update the
        incremental header, whereas nodes with a software data-plane would
        process the pre-allocated header.</t>
      </section>

      <section title="Administrative Boundaries">
        <t>There are several challenges in enabling in-situ OAM in the public
        Internet as well as in corporate/enterprise networks across
        administrative domains, which include but are not limited to:</t>

        <t><list style="symbols">
            <t>Deployment dependent, the data fields that in-situ OAM requires
            as part of a specific transport protocol may not be supported
            across administrative boundaries.</t>

            <t>Current OAM implementations are often done in the slow path,
            i.e., OAM packets are punted to router&rsquo;s CPU for processing.
            This leads to performance and scaling issues and opens up routers
            for attacks such as Denial of Service (DoS) attacks.</t>

            <t>Discovery of network topology and details of the network
            devices across administrative boundaries may open up attack
            vectors compromising network security.</t>

            <t>Specifically on IPv6: At the administrative boundaries IPv6
            packets with extension headers are dropped for several reasons
            described in <xref target="RFC7872"/>.</t>
          </list></t>

        <t>The following considerations will be discussed in a future version
        of this document: If the packet is dropped due to the presence of the
        in-situ OAM; If the policy failure is treated as feature disablement
        and any further recording is stopped but the packet itself is not
        dropped, it may lead to every node in the path to make this policy
        decision.</t>
      </section>

      <section title="Selective Enablement">
        <t>The ability to selectively enable in-situ OAM is valuable. While it
        may be desirable to enable data collection on all traffic or devices,
        this may not always be feasible. In-situ OAM collection may also come
        with a performance impact to forwarding rates or feature capabilities,
        which may be acceptable in only some locations. For example, the SPUD
        prototype uses the notion of "pipes" to describe the portion of the
        traffic that could be subject to in-path inspection. Mechanisms to
        decide which traffic would be subject to in-situ OAM are outside the
        scope of this document.</t>
      </section>

      <section title="Optimization of Node and Interface Identifiers">
        <t>Since packets have a finite maximum size, the data recording or
        carrying capacity of one packet in which the in-situ OAM metadata is
        present is limited. In-situ OAM should use its own dedicated namespace
        (confined to the domain in-situ OAM operates in) to represent node and
        interface IDs to save space in the header. Generic representations of
        node and interface identifiers which are globally unique (such as a
        UUID) would consume significantly more bits of in-situ OAM data.</t>
      </section>

      <section title="Loop Communication Path (IPv6-specifics)">
        <t>When recorded data is required to be analyzed on a source node that
        issues a packet and inserts in-situ OAM data, the recorded data needs
        to be carried back to the source node.</t>

        <t>One way to carry the in-situ OAM data back to the source is to
        utilize an ICMP Echo Request/Reply (ping) or ICMPv6 Echo Request/Reply
        (ping6) mechanism. In order to run the in-situ OAM mechanism
        appropriately on the ping/ping6 mechanism, the following two
        operations should be implemented by the ping/ping6 target node:</t>

        <t><list style="numbers">
            <t>All of the in-situ OAM fields would be copied from an Echo
            Request message to an Echo Reply message.</t>

            <t>The Hop Limit field of the IPv6 header of these messages would
            be copied as a continuous sequence. Further considerations are
            addressed in a future version of this document.</t>
          </list></t>
      </section>
    </section>

    <section title="Requirements for In-situ OAM Data Types">
      <t>The above discussed use cases require different types of in-situ OAM
      data. This section details requirements for in-situ OAM derived from the
      discussion above.</t>

      <section title="Generic Requirements">
        <t><list style="format REQ-G%d:">
            <t>Classification: It should be possible to enable in-situ OAM on
            a selected set of traffic (e.g., per interface, based on an access
            control list specifying a specific set of traffic, etc.) The
            selected set of traffic can also be all traffic.</t>

            <t>Scope: If in-situ OAM is used only within a specific domain,
            provisions need to be put in place to ensure that in-situ OAM data
            stays within the specific domain only.</t>

            <t>Transport independence: Data formats for in-situ OAM shall be
            defined in a transport independent way. In-situ OAM applies to a
            variety of transport protocols. Encapsulations should be defined
            how the generic data formats are carried by a specific
            protocol.</t>

            <t>Layering: It should be possible to have in-situ OAM information
            for different transport protocol layers be present in several
            fields within a single packet. This could for example be the case
            when tunnels are employed and in-situ OAM information is to be
            gathered for both the underlay as well as the overlay network.
            Layering support should not be limited to just underlay and
            overlay, but include more than two layers.</t>

            <t>MTU size: With in-situ OAM information added, packets MUST NOT
            become larger than the path MTU.

<list style="format REQ-G5.%d:">

                <t>If due to some reason a packet which contains in
                situ OAM data record cannot be forwarded due to the presence
                of in-situ OAM data records, the node SHOULD remove the in
                situ OAM data records and forward the packet, rather than drop
                the entire packet.</t>

                <t>If the encapsulating router is unable to insert
                in-situ OAM data records into a packet, e.g., due to MTU
                issues, even though it is configured to do so, it should use
                some operational means to inform the operator (e.g., syslog)
                about the inability to add in-situ OAM data records. Even if
                the in-situ OAM encapsulating node fails to add in-situ OAM
                data records, it should forward the packet normally.</t>

                <t>MTU size consideration for in-situ OAM MUST take
                domain specifics into account, e.g., changes of the domain
                topology due to path protection mechanisms might extend the
                hop count of a path etc.</t>
              </list></t>

            <t>Data structure reuse: The data types and data formats defined
            and used for in-situ OAM ought to be reusable for out-of-band OAM
            telemetry as well.</t>

            <t>Data records format: It is desirable that the format of in-situ
            OAM data-records leverages already defined data formats for OAM as
            much as feasible.</t>

            <t>Combination with active OAM mechanisms: In-situ OAM should be
            useable for active network probing, like for example a customized
            version of traceroute. Decapsulating in-situ OAM nodes may have an
            ability to send the in-situ OAM information retrieved from the
            packet back to the source address of the packet or to the
            encapsulating node.</t>
          </list></t>
      </section>

      <section title="In-situ OAM Data with Per-hop Scope">
        <t><list style="format REQ-H%d:">
            <t>Missing nodes detection: Data shall be present that allows a
            node to detect whether all nodes that might participate in in-situ
            OAM operations have indeed participated.</t>

            <t>Node, instance or device identifier: Data shall be present that
            allows to retrieve the identity of the entity reporting telemetry
            information. The entity can be a device, or a subsystem/component
            within a device. The latter will allow for packet tracing within a
            device in much the same way as between devices.</t>

            <t>Ingress interface identifier: Data shall be present that allows
            the identification of the interface a particular packet was
            received from. The interface can be a logical and/or physical
            entity.</t>

            <t>Egress interface identifier: Data shall be present that allows
            the identification of the interface a particular packet was
            forwarded to. Interface can be a logical or physical entity.</t>

            <t>Time-related requirements<list style="format REQ-H5.%d:">
                <t>Delay: Data shall be present that allows to retrieve the
                delay between two or more points of interest within the
                system. Those points can be within the same device or on
                different devices.</t>

                <t>Jitter: Data shall be present that allows to retrieve the
                jitter between two or more points of interest within the
                system. Those points can be within the same device or on
                different devices. Jitter can be derived from the different
                timestamps gathered and does not necessarily need to be an
                explicit data record.</t>

                <t>Wall-clock time: Data shall be present that allows to
                retrieve the wall-clock time visited a particular point of
                interest in the system.</t>

                <t>Time precision: Time with different precision should be
                supported. Use-case dependent, the required precision could
                e.g., be nanoseconds, microseconds, milliseconds, or
                seconds.</t>
              </list></t>

            <t>Generic data records (like e.g., GPS/Geo-location information):
            It should be possible to add user-defined OAM data at select hops
            to the packet. The semantics of the data are defined by the
            user.</t>
          </list></t>
      </section>

      <section title="In-situ OAM with Selected Hop Scope">
        <t><list style="format REQ-S%d:">
            <t>Proof of transit: Data shall be present which allows to
            securely prove that a packet has visited or ore several particular
            points of interest (i.e., a particular set of nodes). <list
                style="format REQ-S1.%d:">
                <t>In case "Shamir's secret sharing scheme" is used for proof
                of transit, two data records, "random" and "cumulative" shall
                be present. The number of bits used for "random" and
                "cumulative" data records can vary between deployments and
                should thus be configurable.</t>

                <t>Enable a fail-open service chaining system to be converted
                into a fail-closed service chaining system.</t>
              </list></t>
          </list></t>
      </section>

      <section title="In-situ OAM with End-to-end Scope">
        <t><list style="format REQ-E%d:">
            <t>Sequence numbering:<list style="format REQ-E1.%d:">
                <t>Reordering detection: It should be possible to detect
                whether packets have been reordered while traversing an in
                situ OAM domain.</t>

                <t>Duplicates detection: It should be possible to detect
                whether packets have been duplicated while traversing an in
                situ OAM domain.</t>

                <t>Detection of packet drops: It should be possible to detect
                whether packets have been dropped while traversing an in-situ
                OAM domain.</t>
              </list></t>
          </list></t>
      </section>
    </section>

    <!--
    <section title="Manageability Considerations">
      <t>Manageability considerations will be addressed &iacute;n a later
      version of this document.</t>
    </section>
-->

    <section anchor="Security"
             title="Security Considerations and Requirements">
      <t/>

      <section title="General considerations">
        <t>General Security considerations will be expanded on in a later
        version of this document.</t>

        <t>In-situ OAM is considered a "per domain" feature, where one or
        several operators decide on leveraging and configuring in-situ OAM
        according to their needs. Still operators need to properly secure the
        in-situ OAM domain to avoid malicious configuration and use, which
        could include injecting malicious in-situ OAM packets into a
        domain.</t>
      </section>

      <section title="Proof of Transit">
        <t>Threat Model: Attacks on the deployments could be due to malicious
        administrators or accidental misconfiguration resulting in bypassing
        of certain nodes. The solution approach should meet the following
        requirements:</t>

        <t><list style="format REQ-SEC%d:">
            <t>Sound Proof of Transit: A valid and verifiable proof that the
            packet definitively traversed through all the nodes as expected.
            Probabilistic methods to achieve this should be avoided, as the
            same could be exploited by an attacker.</t>

            <t>Tampering of meta data: An active attacker should not be able
            to insert or modify or delete meta data in whole or in parts and
            bypass few (or all) nodes. Any deviation from the expected path
            should be accurately determined.</t>

            <t>Replay Attacks: A attacker (active/passive) should not be able
            to reuse the POT bits in the packet by observing the OAM data in
            the packet, packet characteristics (like IP addresses, octets
            transferred, timestamps) or even the proof bits themselves. The
            solution approach should consider usage of these parameters for
            deriving any secrets cautiously. Mitigating replay attacks beyond
            a window of longer duration could be intractable to achieve with
            fixed number of bits allocated for proof.</t>

            <t>Pre-play Attacks: A active attacker should not be able to
            generate or reuse valid POT bits from legitimate packets, in order
            to prove to the verifier as valid packets. This slight variant of
            replay attacks. The attacker extracts POT bits from legitimate
            packets and ensure they do not reach the verifier. Subsequently
            reuse those POT bits in crafted packets.</t>

            <t>Recycle Secrets: Any configuration of the secrets (like
            cryptographic keys, initialization vectors etc.) either in the
            controller or service functions should be re-configurable.
            Solution approach should enable controls, API calls etc. needed in
            order to perform such recycling. It is desirable to provide
            recommendations on the duration of rotation cycles needed for the
            secure functioning of the overall system.</t>

            <t>Secret storage and distribution: Secrets should be shared with
            the devices over secure channels. Methods should be put in place
            so that secrets cannot be retrieved by non-authorized personnel
            from the devices.</t>
          </list></t>
      </section>
    </section>

    <section anchor="IANA" title="IANA Considerations">
      <t>[RFC Editor: please remove this section prior to publication.]</t>

      <t>This document has no IANA actions.</t>
    </section>

    <section title="Acknowledgements">
      <t>The authors would like to thank Jen Linkova, LJ Wobker, Eric Vyncke,
      Nalini Elkins, Srihari Raghavan, Ranganathan T S, Karthik Babu
      Harichandra Babu, Akshaya Nadahalli, Ignas Bagdonas, LJ Wobker, Erik
      Nordmark, and Andrew Yourtchenko for the comments and advice. This
      document leverages and builds on top of several concepts described in
      <xref target="I-D.kitamura-ipv6-record-route"/>. The authors would like
      to acknowledge the work done by the author Hiroshi Kitamura and people
      involved in writing it.</t>
    </section>
  </middle>

  <!--  *****BACK MATTER ***** -->

  <back>
    <!-- References split into informative and normative -->

    <!-- There are 2 ways to insert reference entries from the citation libraries:
     1. define an ENTITY at the top, and use "ampersand character"RFC2629; here (as shown)
     2. simply use a PI "less than character"?rfc include="reference.RFC.2119.xml"?> here
        (for I-Ds: include="reference.I-D.narten-iana-considerations-rfc2434bis.xml")

     Both are cited textually in the same manner: by using xref elements.
     If you use the PI option, xml2rfc will, by default, try to find included files in the same
     directory as the including file. You can also define the XML_LIBRARY environment variable
     with a value containing a set of directories to search.  These can be either in the local
     filing system or remote ones accessed by http (http://domain/dir/... ).-->

    <references title="Normative References">

      &RFC2119;

    </references>

    <references title="Informative References">
      &RFC791;

      &RFC7665;

      &RFC7799;

      &RFC4884;

      &RFC4950;

      &RFC5837;

      <!--
      &RFC2460;
      &I-D.ietf-sfc-nsh;
      &I-D.ietf-6man-segment-routing-header;
      &I-D.ietf-nvo3-vxlan-gpe;
      &I-D.ietf-ippm-6man-pdm-option;
-->

      &I-D.ietf-spring-segment-routing;

      &I-D.lapukhov-dataplane-probe;

      <reference anchor="I-D.kitamura-ipv6-record-route">
        <front>
          <title>Record Route for IPv6 (PR6) Hop-by-Hop Option
          Extension</title>

          <author fullname="Hiroshi Kitamura" initials="H" surname="Kitamura"/>

          <date month="November" year="2000"/>
        </front>

        <seriesInfo name="Internet-Draft"
                    value="draft-kitamura-ipv6-record-route-00"/>

        <format target="https://tools.ietf.org/id/draft-kitamura-ipv6-record-route-00.txt"
                type="TXT"/>
      </reference>

      &RFC7276;

      &RFC7112;

      &I-D.brockners-proof-of-transit;

      &I-D.brockners-lisp-sr;

      &RFC7872;

      &I-D.SPUD;

      <reference anchor="P4">
        <front>
          <title>P4: In-band Network Telemetry (INT)</title>

          <author fullname="Changhoon Kim" surname="Kim"/>

          <date month="September" year="2015"/>
        </front>
      </reference>

      <!--
      <reference anchor="FD.io" target="https://fd.io/">
        <front>
          <title>Fast Data Project: FD.io</title>

          <author/>

          <date/>
        </front>
      </reference>
-->
    </references>
  </back>
</rfc>
