bgpd: refine the setting up of GTSM

* bgpd.h: Add error code for setting GTSM on iBGP
* bgpd.c: (peer_ttl_security_hops_set) use previous error code and signal
  incompatibility of GTSM+iBGP to vty.
  Consider the session state when setting GTSM, and reset Open/Active peers
  to let them pick up new TTL from start.
diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c
index 5a412f2..ee0cc5d 100644
--- a/bgpd/bgpd.c
+++ b/bgpd/bgpd.c
@@ -4376,7 +4376,7 @@
   zlog_debug ("peer_ttl_security_hops_set: set gtsm_hops to %d for %s", gtsm_hops, peer->host);
 
   if (peer_sort (peer) == BGP_PEER_IBGP)
-    return 0;
+    return BGP_ERR_NO_IBGP_WITH_TTLHACK;
 
   /* We cannot configure ttl-security hops when ebgp-multihop is already
      set.  For non peer-groups, the check is simple.  For peer-groups, it's
@@ -4430,8 +4430,23 @@
 
 	  peer->gtsm_hops = group->conf->gtsm_hops;
 
-	  if (peer->fd >= 0 && peer->gtsm_hops != 0)
-            sockopt_minttl (peer->su.sa.sa_family, peer->fd, MAXTTL + 1 - peer->gtsm_hops);
+	  /* Change setting of existing peer
+	   *   established then change value (may break connectivity)
+	   *   not established yet (teardown session and restart)
+	   *   no session then do nothing (will get handled by next connection)
+	   */
+	  if (peer->status == Established)
+	    {
+	      if (peer->fd >= 0 && peer->gtsm_hops != 0)
+		sockopt_minttl (peer->su.sa.sa_family, peer->fd,
+				MAXTTL + 1 - peer->gtsm_hops);
+	    }
+	  else if (peer->status < Established)
+	    {
+	      if (BGP_DEBUG (events, EVENTS))
+		zlog_debug ("%s Min-ttl changed", peer->host);
+	      BGP_EVENT_ADD (peer, BGP_Stop);
+	    }
 	}
     }