api: rework plain and JSON implementations (#2405)

- match paths through a custom AsyncWebHandler instead of using generic not-found fallback handler
- allow MQTT-like patterns when registering paths (`simple/path`, `path/+/something`, `path/#`)
Replaces `relay/0`, `relay/1` etc. with `relay/+`. Magnitudes are plain paths, but using `/+` in case there's more than 1 magnitude of the same type.
- restore `std::function` as callback container (no more single-byte arg nonsense). Still, limit to 1 type per handler type
- adds JSON handlers which will receive JsonObject root as both input and output. Same logic as plain - GET returns resource data, PUT updates it.
- breaking change to `apiAuthenticate(request)`, it no longer will do `request->send(403)` and expect this to be handled externally.
- allow `Api-Key` header containing the key, works for both GET & PUT plain requests. The only way to set apikey for JSON.
- add `ApiRequest::param` to retrieve both GET and PUT params (aka args), remove ApiBuffer
- remove `API_BUFFER_SIZE`. Allow custom form-data key=value pairs for requests, allow to send basic `String`.
- add `API_JSON_BUFFER_SIZE` for the JSON buffer (both input and output)
- `/apis` replaced with `/api/list`, no longer uses custom handler and is an `apiRegister` callback
- `/api/rpc` custom handler replaced with an `apiRegister` callback

WIP further down:
- no more `webLog` for API requests, unless `webAccessLog` / `WEB_ACCESS_LOG` is set to `1`. This also needs to happen to the other handlers. 
- migrate to ArduinoJson v6, since it become apparent it is actually a good upgrade :)
- actually make use of JSON endpoints more, right now it's just existing GET for sensors and relays
- fork ESPAsyncWebServer to cleanup path parsing and temporary objects attached to the request (also, fix things a lot of things based on PRs there...)
This commit is contained in:
Max Prokhorov
2020-12-05 14:14:38 +03:00
committed by GitHub
parent ac6f0b71e6
commit 8e80a7786c
20 changed files with 1403 additions and 544 deletions

View File

@@ -12,8 +12,6 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
#if API_SUPPORT
#include <vector>
#include "system.h"
#include "web.h"
#include "rpc.h"
@@ -21,252 +19,725 @@ Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
#include <ESPAsyncTCP.h>
#include <ArduinoJson.h>
constexpr size_t ApiPathSizeMax { 64ul };
std::vector<Api> _apis;
#include <algorithm>
#include <cstring>
#include <forward_list>
#include <vector>
// -----------------------------------------------------------------------------
// API
// -----------------------------------------------------------------------------
bool _asJson(AsyncWebServerRequest *request) {
bool asJson = false;
if (request->hasHeader("Accept")) {
AsyncWebHeader* h = request->getHeader("Accept");
asJson = h->value().equals("application/json");
}
return asJson;
}
void _onAPIsText(AsyncWebServerRequest *request) {
AsyncResponseStream *response = request->beginResponseStream("text/plain");
char buffer[ApiPathSizeMax] = {0};
for (auto& api : _apis) {
sprintf_P(buffer, PSTR("/api/%s\n"), api.path.c_str());
response->write(buffer);
}
request->send(response);
}
constexpr size_t ApiJsonBufferSize = 1024;
void _onAPIsJson(AsyncWebServerRequest *request) {
DynamicJsonBuffer jsonBuffer(ApiJsonBufferSize);
JsonArray& root = jsonBuffer.createArray();
char buffer[ApiPathSizeMax] = {0};
for (auto& api : _apis) {
sprintf(buffer, "/api/%s", api.path.c_str());
root.add(buffer);
PathParts::PathParts(const String& path) :
_path(path)
{
if (!_path.length()) {
_ok = false;
return;
}
AsyncResponseStream *response = request->beginResponseStream("application/json");
root.printTo(*response);
request->send(response);
PathPart::Type type { PathPart::Type::Unknown };
size_t length { 0ul };
size_t offset { 0ul };
}
void _onAPIs(AsyncWebServerRequest *request) {
webLog(request);
if (!apiAuthenticate(request)) return;
bool asJson = _asJson(request);
String output;
if (asJson) {
_onAPIsJson(request);
} else {
_onAPIsText(request);
const char* p { _path.c_str() };
if (*p == '\0') {
goto error;
}
_parts.reserve(std::count(_path.begin(), _path.end(), '/') + 1);
start:
type = PathPart::Type::Unknown;
length = 0;
offset = p - _path.c_str();
switch (*p) {
case '+':
goto parse_single_wildcard;
case '#':
goto parse_multi_wildcard;
case '/':
default:
goto parse_value;
}
parse_value:
type = PathPart::Type::Value;
switch (*p) {
case '+':
case '#':
goto error;
case '/':
case '\0':
goto push_result;
}
++p;
++length;
goto parse_value;
parse_single_wildcard:
type = PathPart::Type::SingleWildcard;
++p;
switch (*p) {
case '/':
++p;
case '\0':
goto push_result;
}
goto error;
parse_multi_wildcard:
type = PathPart::Type::MultiWildcard;
++p;
if (*p == '\0') {
goto push_result;
}
goto error;
push_result:
emplace_back(type, offset, length);
if (*p == '/') {
++p;
goto start;
} else if (*p != '\0') {
goto start;
}
goto success;
error:
_ok = false;
_parts.clear();
return;
success:
_ok = true;
}
void _onRPC(AsyncWebServerRequest *request) {
// match when, for example, given the path 'topic/one/two/three' and pattern 'topic/+/two/+'
webLog(request);
if (!apiAuthenticate(request)) return;
bool PathParts::match(const PathParts& path) const {
if (!_ok || !path) {
return false;
}
//bool asJson = _asJson(request);
int response = 404;
auto lhs = begin();
auto rhs = path.begin();
if (request->hasParam("action")) {
auto lhs_end = end();
auto rhs_end = path.end();
AsyncWebParameter* p = request->getParam("action");
loop:
if (lhs == lhs_end) {
goto check_end;
}
const auto action = p->value();
DEBUG_MSG_P(PSTR("[RPC] Action: %s\n"), action.c_str());
if (rpcHandleAction(action)) {
response = 204;
switch ((*lhs).type) {
case PathPart::Type::Value:
if (
(rhs != rhs_end)
&& ((*rhs).type == PathPart::Type::Value)
&& ((*rhs).offset == (*lhs).offset)
&& ((*rhs).length == (*lhs).length)
) {
if (0 == std::memcmp(
_path.c_str() + (*lhs).offset,
path.path().c_str() + (*rhs).offset,
(*rhs).length))
{
std::advance(lhs, 1);
std::advance(rhs, 1);
goto loop;
}
}
goto error;
case PathPart::Type::SingleWildcard:
if (
(rhs != rhs_end)
&& ((*rhs).type == PathPart::Type::Value)
) {
std::advance(lhs, 1);
std::advance(rhs, 1);
goto loop;
}
goto error;
case PathPart::Type::MultiWildcard:
if (std::next(lhs) == lhs_end) {
while (rhs != rhs_end) {
if ((*rhs).type != PathPart::Type::Value) {
goto error;
}
std::advance(rhs, 1);
}
lhs = lhs_end;
break;
}
goto error;
case PathPart::Type::Unknown:
goto error;
};
check_end:
if ((lhs == lhs_end) && (rhs == rhs_end)) {
return true;
}
request->send(response);
error:
return false;
}
struct ApiMatch {
Api* api { nullptr };
Api::Type type { Api::Type::Basic };
};
String ApiRequest::wildcard(int index) const {
if (index < 0) {
index = std::abs(index + 1);
}
ApiMatch _apiMatch(const String& url, AsyncWebServerRequest* request) {
if (std::abs(index) >= _pattern.parts().size()) {
return _empty_string();
}
ApiMatch result;
char buffer[ApiPathSizeMax] = {0};
int counter { 0 };
auto& pattern = _pattern.parts();
for (auto& api : _apis) {
sprintf_P(buffer, PSTR("/api/%s"), api.path.c_str());
if (url != buffer) {
continue;
for (unsigned int part = 0; part < pattern.size(); ++part) {
auto& lhs = pattern[part];
if (PathPart::Type::SingleWildcard == lhs.type) {
if (counter == index) {
auto& rhs = _parts.parts()[part];
return _parts.path().substring(rhs.offset, rhs.offset + rhs.length);
}
++counter;
}
}
auto type = _asJson(request)
? Api::Type::Json
: Api::Type::Basic;
return _empty_string();
}
result.api = &api;
result.type = type;
break;
size_t ApiRequest::wildcards() const {
size_t result { 0ul };
for (auto& part : _pattern) {
if (PathPart::Type::SingleWildcard == part.type) {
++result;
}
}
return result;
}
bool _apiDispatchRequest(const String& url, AsyncWebServerRequest* request) {
// -----------------------------------------------------------------------------
bool _apiAccepts(AsyncWebServerRequest* request, const __FlashStringHelper* str) {
auto* header = request->getHeader(F("Accept"));
if (header) {
return
(header->value().indexOf(F("*/*")) >= 0)
|| (header->value().indexOf(str) >= 0);
}
return false;
}
bool _apiAcceptsText(AsyncWebServerRequest* request) {
return _apiAccepts(request, F("text/plain"));
}
bool _apiAcceptsJson(AsyncWebServerRequest* request) {
return _apiAccepts(request, F("application/json"));
}
bool _apiMatchHeader(AsyncWebServerRequest* request, const __FlashStringHelper* key, const __FlashStringHelper* value) {
auto* header = request->getHeader(key);
if (header) {
return header->value().equals(value);
}
return false;
}
bool _apiIsJsonContent(AsyncWebServerRequest* request) {
return _apiMatchHeader(request, F("Content-Type"), F("application/json"));
}
bool _apiIsFormDataContent(AsyncWebServerRequest* request) {
return _apiMatchHeader(request, F("Content-Type"), F("application/x-www-form-urlencoded"));
}
struct ApiRequestHelper {
ApiRequestHelper(const ApiRequestHelper&) = delete;
ApiRequestHelper(ApiRequestHelper&&) noexcept = default;
// &path is expected to be request->url(), which is valid throughout the request's lifetime
explicit ApiRequestHelper(AsyncWebServerRequest& request, const PathParts& pattern) :
_request(request),
_pattern(pattern),
_path(request.url()),
_match(_pattern.match(_path))
{}
ApiRequest request() const {
return ApiRequest(_request, _pattern, _path);
}
const PathParts& parts() const {
return _path;
}
bool match() const {
return _match;
}
private:
AsyncWebServerRequest& _request;
const PathParts& _pattern;
PathParts _path;
bool _match;
};
// Because the webserver request is split between multiple separate function invocations, we need to preserve some state.
// TODO: in case we are dealing with multicore, perhaps enforcing static-size data structs instead of the vector would we better,
// to avoid calling generic malloc when paths are parsed?
//
// Some quirks to deal with:
// - handleBody is called before handleRequest, and there's no way to signal completion / success of both callbacks to the server
// - Server never checks for request closing in filter or canHandle, so if we don't want to handle large content-length, it
// will still flow through the lwip backend.
// - `request->_tempObject` is used to keep API request state, but it's just a plain void pointer
// - espasyncwebserver will `free(_tempObject)` when request is disconnected, but only after this callbackhandler is done.
// make sure it's set to nullptr via `AsyncWebServerRequest::onDisconnect`
// - ALL headers are parsed (and we could access those during filter and canHandle callbacks), but we need to explicitly
// request them to stay in memory so that the actual handler can work with them
void _apiAttachHelper(AsyncWebServerRequest& request, ApiRequestHelper&& helper) {
request._tempObject = new ApiRequestHelper(std::move(helper));
request.onDisconnect([&]() {
auto* ptr = reinterpret_cast<ApiRequestHelper*>(request._tempObject);
delete ptr;
request._tempObject = nullptr;
});
request.addInterestingHeader(F("Api-Key"));
}
class ApiBaseWebHandler : public AsyncWebHandler {
public:
ApiBaseWebHandler() = delete;
ApiBaseWebHandler(const ApiBaseWebHandler&) = delete;
ApiBaseWebHandler(ApiBaseWebHandler&&) = delete;
// In case this needs to be copied or moved, ensure PathParts copy references the new object's string
template <typename Pattern>
explicit ApiBaseWebHandler(Pattern&& pattern) :
_pattern(std::forward<Pattern>(pattern)),
_parts(_pattern)
{}
const String& pattern() const {
return _pattern;
}
const PathParts& parts() const {
return _parts;
}
private:
String _pattern;
PathParts _parts;
};
// 'Modernized' API configuration:
// - `Api-Key` header for both GET and PUT
// - Parse request body as JSON object. Limited to LWIP internal buffer size, and will also break when client
// does weird stuff and PUTs data in multiple packets b/c only the initial packet is parsed.
// - Same as the text/plain, when ApiRequest::handle was not called it will then call GET
//
// TODO: bump to arduinojson v6 to handle partial / broken data payloads
// TODO: somehow detect partial data and buffer (optionally)
// TODO: POST instead of PUT?
class ApiJsonWebHandler final : public ApiBaseWebHandler {
public:
static constexpr size_t BufferSize { API_JSON_BUFFER_SIZE };
struct ReadOnlyStream : public Stream {
ReadOnlyStream() = delete;
explicit ReadOnlyStream(const uint8_t* buffer, size_t size) :
_buffer(buffer),
_size(size)
{}
int available() override {
return _size - _index;
}
int peek() override {
if (_index < _size) {
return static_cast<int>(_buffer[_index]);
}
return -1;
}
int read() override {
auto peeked = peek();
if (peeked >= 0) {
++_index;
}
return peeked;
}
// since we are fixed in size, no need for any timeouts and the only available option is to return full chunk of data
size_t readBytes(uint8_t* ptr, size_t size) override {
if ((_index < _size) && ((_size - _index) >= size)) {
std::copy(_buffer + _index, _buffer + _index + size, ptr);
_index += size;
return size;
}
return 0;
}
size_t readBytes(char* ptr, size_t size) override {
return readBytes(reinterpret_cast<uint8_t*>(ptr), size);
}
void flush() override {
}
size_t write(const uint8_t*, size_t) override {
return 0;
}
size_t write(uint8_t) override {
return 0;
}
const uint8_t* _buffer;
const size_t _size;
size_t _index { 0 };
};
ApiJsonWebHandler() = delete;
ApiJsonWebHandler(const ApiJsonWebHandler&) = delete;
ApiJsonWebHandler(ApiJsonWebHandler&&) = delete;
template <typename Path, typename Callback>
ApiJsonWebHandler(Path&& path, Callback&& get, Callback&& put) :
ApiBaseWebHandler(std::forward<Path>(path)),
_get(std::forward<Callback>(get)),
_put(std::forward<Callback>(put))
{}
bool isRequestHandlerTrivial() override {
return true;
}
bool canHandle(AsyncWebServerRequest* request) override {
if (!apiEnabled()) {
return false;
}
if (!_apiAcceptsJson(request)) {
return false;
}
auto helper = ApiRequestHelper(*request, parts());
if (helper.match() && apiAuthenticate(request)) {
switch (request->method()) {
case HTTP_HEAD:
return true;
case HTTP_PUT:
if (!_apiIsJsonContent(request)) {
return false;
}
if (!_put) {
return false;
}
case HTTP_GET:
if (!_get) {
return false;
}
break;
default:
return false;
}
_apiAttachHelper(*request, std::move(helper));
return true;
}
auto match = _apiMatch(url, request);
if (!match.api) {
return false;
}
if (match.type != match.api->type) {
DEBUG_MSG_P(PSTR("[API] Cannot handle the request type\n"));
request->send(404);
return true;
}
const bool is_put = (
(!apiRestFul() || (request->method() == HTTP_PUT))
&& request->hasParam("value", request->method() == HTTP_PUT)
);
ApiBuffer buffer;
switch (match.api->type) {
case Api::Type::Basic: {
if (!match.api->get.basic) {
break;
}
if (is_put) {
if (!match.api->put.basic) {
break;
}
auto value = request->getParam("value", request->method() == HTTP_PUT)->value();
if (buffer.size < (value.length() + 1ul)) {
break;
}
std::copy(value.c_str(), value.c_str() + value.length() + 1, buffer.data);
match.api->put.basic(*match.api, buffer);
buffer.erase();
}
match.api->get.basic(*match.api, buffer);
request->send(200, "text/plain", buffer.data);
return true;
}
// TODO: pass the body instead of `value` param
// TODO: handle HTTP_PUT
case Api::Type::Json: {
if (!match.api->get.json || is_put) {
break;
}
DynamicJsonBuffer jsonBuffer(API_BUFFER_SIZE);
void _handleGet(AsyncWebServerRequest* request, ApiRequest& apireq) {
DynamicJsonBuffer jsonBuffer(API_JSON_BUFFER_SIZE);
JsonObject& root = jsonBuffer.createObject();
if (!_get(apireq, root)) {
request->send(500);
return;
}
match.api->get.json(*match.api, root);
if (!apireq.done()) {
AsyncResponseStream *response = request->beginResponseStream("application/json", root.measureLength() + 1);
root.printTo(*response);
request->send(response);
return;
}
AsyncResponseStream *response = request->beginResponseStream("application/json", root.measureLength() + 1);
root.printTo(*response);
request->send(response);
return true;
request->send(500);
}
void _handlePut(AsyncWebServerRequest* request, uint8_t* data, size_t size) {
// XXX: arduinojson v5 de-serializer will happily read garbage from raw ptr, since there's no length limit
// this is fixed in v6 though. for now, use a wrapper, but be aware that this actually uses more mem for the jsonbuffer
DynamicJsonBuffer jsonBuffer(API_JSON_BUFFER_SIZE);
ReadOnlyStream stream(data, size);
JsonObject& root = jsonBuffer.parseObject(stream);
if (!root.success()) {
request->send(500);
return;
}
auto& helper = *reinterpret_cast<ApiRequestHelper*>(request->_tempObject);
auto apireq = helper.request();
if (!_put(apireq, root)) {
request->send(500);
return;
}
if (!apireq.done()) {
_handleGet(request, apireq);
}
return;
}
DEBUG_MSG_P(PSTR("[API] Method not supported\n"));
request->send(405);
return true;
}
bool _apiRequestCallback(AsyncWebServerRequest* request) {
String url = request->url();
if (url.equals("/rpc")) {
_onRPC(request);
return true;
void handleBody(AsyncWebServerRequest* request, uint8_t* data, size_t len, size_t, size_t total) override {
if (total && (len == total)) {
_handlePut(request, data, total);
}
}
if (url.equals("/api") || url.equals("/apis")) {
_onAPIs(request);
return true;
void handleRequest(AsyncWebServerRequest* request) override {
auto& helper = *reinterpret_cast<ApiRequestHelper*>(request->_tempObject);
switch (request->method()) {
case HTTP_HEAD:
request->send(204);
return;
case HTTP_GET: {
auto apireq = helper.request();
_handleGet(request, apireq);
return;
}
// see handleBody()
case HTTP_PUT:
break;
default:
request->send(405);
break;
}
}
if (!url.startsWith("/api/")) return false;
const String& pattern() const {
return ApiBaseWebHandler::pattern();
}
// [alexa] don't call the http api -> response for alexa is done by fauxmoesp lib
#if ALEXA_SUPPORT
if (url.indexOf("/lights") > 14 ) return false;
#endif
const PathParts& parts() const {
return ApiBaseWebHandler::parts();
}
if (!apiAuthenticate(request)) return false;
private:
ApiJsonHandler _get;
ApiJsonHandler _put;
};
return _apiDispatchRequest(url, request);
// ESPurna legacy API configuration
// - ?apikey=... to authorize in GET or PUT
// - ?anything=... for input data (common key is "value")
// MUST correctly override isRequestHandlerTrivial() to allow auth with PUT
// (i.e. so that ESPAsyncWebServer parses the body and adds form-data to request params list)
}
class ApiBasicWebHandler final : public ApiBaseWebHandler {
public:
template <typename Path, typename Callback>
ApiBasicWebHandler(Path&& path, Callback&& get, Callback&& put) :
ApiBaseWebHandler(std::forward<Path>(path)),
_get(std::forward<Callback>(get)),
_put(std::forward<Callback>(put))
{}
bool isRequestHandlerTrivial() override {
return false;
}
bool canHandle(AsyncWebServerRequest* request) override {
if (!apiEnabled()) {
return false;
}
if (!_apiAcceptsText(request)) {
return false;
}
switch (request->method()) {
case HTTP_HEAD:
case HTTP_GET:
break;
case HTTP_PUT:
if (!_apiIsFormDataContent(request)) {
return false;
}
break;
default:
return false;
}
auto helper = ApiRequestHelper(*request, parts());
if (helper.match()) {
_apiAttachHelper(*request, std::move(helper));
return true;
}
return false;
}
void handleRequest(AsyncWebServerRequest* request) override {
if (!apiAuthenticate(request)) {
request->send(403);
return;
}
auto method = request->method();
const bool is_put = (
(!apiRestFul()|| (HTTP_PUT == method))
&& request->hasParam("value", HTTP_PUT == method)
);
switch (method) {
case HTTP_HEAD:
request->send(204);
return;
case HTTP_GET:
case HTTP_PUT: {
auto& helper = *reinterpret_cast<ApiRequestHelper*>(request->_tempObject);
auto apireq = helper.request();
if (is_put) {
if (!_put(apireq)) {
request->send(500);
return;
}
if (apireq.done()) {
return;
}
}
if (!_get(apireq)) {
request->send(500);
return;
}
if (!apireq.done()) {
request->send(204);
return;
}
}
default:
request->send(405);
return;
}
}
const ApiBasicHandler& get() const {
return _get;
}
const ApiBasicHandler& put() const {
return _put;
}
const String& pattern() const {
return ApiBaseWebHandler::pattern();
}
const PathParts& parts() const {
return ApiBaseWebHandler::parts();
}
private:
ApiBasicHandler _get;
ApiBasicHandler _put;
};
// -----------------------------------------------------------------------------
void apiReserve(size_t size) {
_apis.reserve(_apis.size() + size);
namespace {
std::forward_list<ApiBaseWebHandler*> _apis;
template <typename Handler, typename Callback>
void _apiRegister(const String& path, Callback&& get, Callback&& put) {
// `String` is a given, since we *do* need to construct this dynamically in sensors
auto* ptr = new Handler(String(F(API_BASE_PATH)) + path, std::forward<Callback>(get), std::forward<Callback>(put));
webServer().addHandler(reinterpret_cast<AsyncWebHandler*>(ptr));
_apis.emplace_front(ptr);
}
void apiRegister(const Api& api) {
if (api.path.length() >= (ApiPathSizeMax - strlen("/api/") - 1ul)) {
return;
}
_apis.push_back(api);
} // namespace
void apiRegister(const String& path, ApiBasicHandler&& get, ApiBasicHandler&& put) {
_apiRegister<ApiBasicWebHandler>(path, std::move(get), std::move(put));
}
void apiRegister(const String& path, ApiJsonHandler&& get, ApiJsonHandler&& put) {
_apiRegister<ApiJsonWebHandler>(path, std::move(get), std::move(put));
}
void apiSetup() {
webRequestRegister(_apiRequestCallback);
apiRegister(F("list"),
[](ApiRequest& request) {
String paths;
for (auto& api : _apis) {
paths += api->pattern() + "\r\n";
}
request.send(paths);
return true;
},
nullptr
);
apiRegister(F("rpc"),
nullptr,
[](ApiRequest& request) {
if (rpcHandleAction(request.param(F("action")))) {
return apiOk(request);
}
return apiError(request);
}
);
}
void apiOk(const Api&, ApiBuffer& buffer) {
buffer.data[0] = 'O';
buffer.data[1] = 'K';
buffer.data[2] = '\0';
bool apiOk(ApiRequest& request) {
request.send(F("OK"));
return true;
}
void apiError(const Api&, ApiBuffer& buffer) {
buffer.data[0] = '-';
buffer.data[1] = 'E';
buffer.data[2] = 'R';
buffer.data[3] = 'R';
buffer.data[4] = 'O';
buffer.data[5] = 'R';
buffer.data[6] = '\0';
bool apiError(ApiRequest& request) {
request.send(F("ERROR"));
return true;
}
#endif // API_SUPPORT