Skip to content

Commit 9da8afe

Browse files
n0099n0099
committed
Merge pull request #100 from n0099/union-post-queries
2 parents bb39f2b + 8927f10 commit 9da8afe

14 files changed

Lines changed: 174 additions & 72 deletions

‎be/src/Controller/PostsController.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public function query(Request $request): array
101101
),
102102
'users' => $users,
103103
'latestRepliers' => $latestRepliers,
104-
'queries' => $this->query->queryResult->queries,
104+
'query' => $this->query->queryResult->query,
105105
];
106106
}
107107
}

‎be/src/DTO/PostKey/BasePostKey.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
public function __construct(
88
public int $fid,
99
public int $postId,
10-
public string $orderByFieldName,
1110
public mixed $orderByFieldValue,
1211
) {}
1312
}

‎be/src/DTO/PostKey/PostKeyWithParent.php‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,8 @@ public function __construct(
88
int $fid,
99
public int $parentPostId,
1010
int $postId,
11-
string $orderByFieldName,
1211
mixed $orderByFieldValue,
1312
) {
14-
parent::__construct($fid, $postId, $orderByFieldName, $orderByFieldValue);
13+
parent::__construct($fid, $postId, $orderByFieldValue);
1514
}
1615
}

‎be/src/DTO/PostKey/SubReply.php‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,8 @@ public function __construct(
99
public int $tid,
1010
int $parentPostId,
1111
int $postId,
12-
string $orderByFieldName,
1312
mixed $orderByFieldValue,
1413
) {
15-
parent::__construct($fid, $parentPostId, $postId, $orderByFieldName, $orderByFieldValue);
14+
parent::__construct($fid, $parentPostId, $postId, $orderByFieldValue);
1615
}
1716
}

‎be/src/Doctrine/InterpolateParametersSQLOutputWalker.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<?php
2+
3+
namespace App\Doctrine;
4+
5+
use Doctrine\DBAL\ArrayParameterType;
6+
use Doctrine\DBAL\ParameterType;
7+
use Doctrine\DBAL\Types\BooleanType;
8+
use Doctrine\DBAL\Types\Exception\ValueNotConvertible;
9+
use Doctrine\DBAL\Types\Type;
10+
use Doctrine\ORM\Query\AST;
11+
use Doctrine\ORM\Query\SqlOutputWalker;
12+
13+
/** @see https://stackoverflow.com/questions/2095394/doctrine-how-to-print-out-the-real-sql-not-just-the-prepared-statement/79647754#79647754 */
14+
class InterpolateParametersSQLOutputWalker extends SqlOutputWalker
15+
{
16+
/** {@inheritdoc} */
17+
public function walkInputParameter(AST\InputParameter $inputParam): string
18+
{
19+
$parameter = $this->getQuery()->getParameter($inputParam->name);
20+
if ($parameter === null) {
21+
return '?';
22+
}
23+
24+
$value = $parameter->getValue();
25+
/** @var ParameterType|ArrayParameterType|int|string $typeName */
26+
/** @see \Doctrine\ORM\Query\ParameterTypeInferer::inferType() */
27+
$typeName = $parameter->getType();
28+
$platform = $this->getConnection()->getDatabasePlatform();
29+
$processParameterType = static fn(ParameterType $type) => static fn($value): string =>
30+
(match ($type) { /** @see Type::getBindingType() */
31+
ParameterType::NULL => 'NULL',
32+
ParameterType::INTEGER => $value,
33+
ParameterType::BOOLEAN => (new BooleanType())->convertToDatabaseValue($value, $platform),
34+
ParameterType::STRING, ParameterType::ASCII => $platform->quoteStringLiteral($value),
35+
default => throw new ValueNotConvertible($value, $type->name)
36+
});
37+
38+
if (is_string($typeName) && Type::hasType($typeName)) {
39+
return Type::getType($typeName)->convertToDatabaseValue($value, $platform);
40+
}
41+
if ($typeName instanceof ParameterType) {
42+
return $processParameterType($typeName)($value);
43+
}
44+
if ($typeName instanceof ArrayParameterType && is_array($value)) {
45+
$type = ArrayParameterType::toElementParameterType($typeName);
46+
return implode(', ', array_map($processParameterType($type), $value));
47+
}
48+
49+
throw new ValueNotConvertible($value, $typeName);
50+
}
51+
}

‎be/src/PostsQuery/CursorCodec.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414
* threads: Collection<Thread>,
1515
* replies: Collection<Reply>,
1616
* subReplies: Collection<SubReply>,
17-
* } */
17+
* }
18+
*/
1819
class CursorCodec
1920
{
2021
/** @param PostsKeyByTypePluralName $postsKeyByTypePluralName */

‎be/src/PostsQuery/Query.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ private function buildQueries(QueryParams $params, ?int $fid): Collection
6262
return collect($this->postRepositoryFactory->newForumPosts())
6363
->only($params->getUniqueParamValue('postTypes'))
6464
->map(function (PostRepository $repository) use ($fid, $params, &$cachedUserQueryResult): QueryBuilder {
65-
$query = $repository->selectPostKeyDTO($this->getOrderByField());
65+
$query = $repository->selectUnionPostKey();
6666
if ($fid !== null) {
6767
$query = $query->where('t.fid = :fid')->setParameter('fid', $fid);
6868
}

‎be/src/PostsQuery/QueryResult.php‎

Lines changed: 104 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,17 @@
22

33
namespace App\PostsQuery;
44

5+
use App\Doctrine\InterpolateParametersSQLOutputWalker;
56
use App\DTO\PostKey\Reply as ReplyKey;
67
use App\DTO\PostKey\SubReply as SubReplyKey;
78
use App\DTO\PostKey\Thread as ThreadKey;
89
use App\Helper;
9-
use Doctrine\DBAL\ParameterType;
10+
use Doctrine\DBAL\Query\QueryBuilder as DBALQueryBuilder;
11+
use Doctrine\DBAL\Query\UnionType;
12+
use Doctrine\ORM\AbstractQuery;
1013
use Doctrine\ORM\Query\Expr\Comparison;
11-
use Doctrine\ORM\Query\Parameter;
14+
use Doctrine\ORM\Query\Parser;
15+
use Doctrine\ORM\Query\ResultSetMapping;
1216
use Doctrine\ORM\QueryBuilder;
1317
use Illuminate\Support\Collection;
1418
use Symfony\Component\DependencyInjection\ParameterBag\ContainerBagInterface;
@@ -32,7 +36,7 @@
3236

3337
public ?string $nextCursor;
3438

35-
public Collection $queries;
39+
public array $query;
3640

3741
public function __construct(
3842
private Stopwatch $stopwatch,
@@ -41,23 +45,12 @@ public function __construct(
4145
private int $perPageItems = 50,
4246
) {}
4347

44-
/** @return array{result: Collection, hasMorePages: bool, query: string, queryPlan: array} */
45-
public function getQueryResult(QueryBuilder $queryBuilder, int $limit): array
48+
/** @return array{result: Collection, hasMorePages: bool, queryPlan: array} */
49+
public function getQueryResult(AbstractQuery $query, int $limit): array
4650
{
4751
$maxResults = $limit + 1;
48-
$query = $queryBuilder->setMaxResults($maxResults)->getQuery();
49-
50-
$entityManager = $query->getEntityManager();
51-
$connection = $entityManager->getConnection();
52-
$parameters = collect($query->getParameters())->mapWithKeys(static fn(Parameter $p) => [
53-
':' . $p->getName() => match($p->getType()) {
54-
ParameterType::STRING => $connection->getDatabasePlatform()->quoteStringLiteral($p->getValue()),
55-
default => $p->getValue(),
56-
}
57-
]);
58-
$rawSQL = $entityManager->createQuery(strtr($query->getDQL(), $parameters->toArray()))->getSQL();
59-
$explainJSON = \Safe\json_decode($connection->executeQuery(
60-
'EXPLAIN (COSTS, VERBOSE, BUFFERS, FORMAT JSON) ' . $rawSQL . " LIMIT $maxResults"
52+
$explainJSON = \Safe\json_decode($query->getEntityManager()->getConnection()->executeQuery(
53+
'EXPLAIN (COSTS, VERBOSE, BUFFERS, FORMAT JSON) ' . $query->getSQL()
6154
)->fetchOne(), true);
6255
$plansCost = array_sum(array_map(static fn(array $plan) => $plan['Plan']['Total Cost'], $explainJSON));
6356
$planCostLimit = $this->containerBag->get('app.query_plan_cost_limit');
@@ -73,7 +66,6 @@ public function getQueryResult(QueryBuilder $queryBuilder, int $limit): array
7366
return [
7467
'result' => $result,
7568
'hasMorePages' => $hasMorePages ?? false,
76-
'query' => $rawSQL,
7769
'queryPlan' => $explainJSON
7870
];
7971
}
@@ -94,16 +86,19 @@ public function setResult(
9486
// remove query for post type with an empty encoded cursor ',,'
9587
$queries = $queries->intersectByKeys($cursorsKeyByPostType);
9688
}
89+
$maxResults = $this->perPageItems + 1;
9790

98-
$queries->each(function (QueryBuilder $qb, string $postType) use ($isOrderByDesc, $orderByField, $cursorsKeyByPostType) {
99-
$qb->addOrderBy("t.$orderByField", $isOrderByDesc === true ? 'DESC' : 'ASC')
91+
$queries->each(function (QueryBuilder $qb, string $postType) use ($maxResults, $isOrderByDesc, $orderByField, $cursorsKeyByPostType) {
92+
$qb->addSelect("t.$orderByField AS orderByField")
93+
->addOrderBy("t.$orderByField", $isOrderByDesc === true ? 'DESC' : 'ASC')
10094
// cursor paginator requires values of orderBy column are unique
10195
// if not it should fall back to other unique field (here is the post ID primary key)
10296
// https://use-the-index-luke.com/no-offset
10397
// https://mysql.rjweb.org/doc.php/pagination
10498
// https://medium.com/swlh/how-to-implement-cursor-pagination-like-a-pro-513140b65f32
10599
// https://slack.engineering/evolving-api-pagination-at-slack/
106-
->addOrderBy('t.' . Helper::POST_TYPE_TO_ID[$postType]);
100+
->addOrderBy('t.' . Helper::POST_TYPE_TO_ID[$postType])
101+
->setMaxResults($maxResults);
107102

108103
$cursors = $cursorsKeyByPostType->get($postType, collect());
109104
if ($cursors->isEmpty()) {
@@ -118,16 +113,12 @@ public function setResult(
118113
$cursors->mapWithKeys(fn($fieldValue, string $fieldName) =>
119114
$qb->setParameter("cursor_$fieldName", $fieldValue)); // prevent overwriting existing param
120115
});
121-
122-
$results = $queries->map(fn(QueryBuilder $query) =>
123-
self::getQueryResult($query, $this->perPageItems));
124-
$results = $queries->map(fn(QueryBuilder $queryBuilder) =>
125-
$this->getQueryResult($queryBuilder, $this->perPageItems));
126-
/** @var PostsKeyByTypePluralName $postsKeyByTypePluralName */
127-
$postsKeyByTypePluralName = $results
128-
->mapWithKeys(fn(array $tuple, string $postType) =>
129-
[Helper::POST_TYPE_TO_PLURAL[$postType] => $tuple['result']]);
130-
Helper::abortAPIIf(40401, $postsKeyByTypePluralName->every(static fn(Collection $i) => $i->isEmpty()));
116+
[
117+
'unionOfQueriesSQL' => $unionOfQueriesSQL,
118+
'postsKeyByTypePluralName' => $postsKeyByTypePluralName,
119+
'hasMorePages' => $hasMorePages,
120+
'queryPlan' => $queryPlan
121+
] = $this->getUnionQueryResult($queries, $isOrderByDesc, $maxResults);
131122

132123
$this->threads = $postsKeyByTypePluralName->get('threads', collect());
133124
$this->replies = $postsKeyByTypePluralName->get('replies', collect());
@@ -136,16 +127,92 @@ public function setResult(
136127
?? $this->replies->first()->fid
137128
?? $this->subReplies->first()->fid;
138129
$this->currentCursor = $cursorParamValue ?? '';
139-
$this->nextCursor = $results->pluck('hasMorePages')
140-
->contains(static fn(bool $hasMorePages) => $hasMorePages)
130+
$this->nextCursor = $hasMorePages
141131
? $this->cursorCodec->encodeNextCursor($postsKeyByTypePluralName->except(
142132
$queryByPostIDParamsName->map(static fn(string $postID) => Helper::POST_ID_TO_TYPE_PLURAL[$postID])
143133
))
144134
: null;
145-
$this->queries = $results->mapWithKeys(fn(array $tuple, string $postType) =>
146-
[$postType => ['query' => $tuple['query'], 'plan' => $tuple['queryPlan']]]
147-
);
135+
$this->query = ['query' => $unionOfQueriesSQL, 'plan' => $queryPlan];
148136

149137
$this->stopwatch->stop('setResult');
150138
}
139+
140+
/**
141+
* @psalm-type UnionPostKey = array{
142+
* postType: 'reply'|'subReply'|'thread',
143+
* postId: int,
144+
* fid: int,
145+
* tid: int,
146+
* pid: int,
147+
* orderByField: mixed
148+
* }
149+
* @param Collection<Helpecr::POST_TYPE, QueryBuilder> $queries
150+
* @param bool $isOrderByDesc
151+
* @param int $maxResults
152+
* @return array{unionOfQueriesSQL: string, postsKeyByTypePluralName: PostsKeyByTypePluralName, hasMorePages: bool, queryPlan: array}
153+
*/
154+
private function getUnionQueryResult(Collection $queries, bool $isOrderByDesc, int $maxResults): array
155+
{
156+
/** @var DBALQueryBuilder $unionOfQueries */
157+
// https://stackoverflow.com/questions/36959801/doctrine-orm-querybuilder-or-dbal-querybuilder
158+
$unionOfQueries = $queries->reduce(function (?DBALQueryBuilder $dbalQueryBuilder, QueryBuilder $ormQueryBuilder) {
159+
$ormQuery = $ormQueryBuilder->getQuery();
160+
$ormQuery->setHint(\Doctrine\ORM\Query::HINT_CUSTOM_OUTPUT_WALKER, InterpolateParametersSQLOutputWalker::class);
161+
$sql = $ormQuery->getSQL();
162+
if ($dbalQueryBuilder === null) {
163+
return $ormQueryBuilder->getEntityManager()->getConnection()
164+
->createQueryBuilder()->union($sql);
165+
}
166+
return $dbalQueryBuilder->addUnion($sql, UnionType::ALL);
167+
});
168+
$firstQuery = $queries->first();
169+
170+
/** @var array{key-of<UnionPostKey>, string} $firstQueryFieldAliases */
171+
// field name and aliases in the first query in a union will override any other queries in union
172+
$firstQueryFieldAliases = array_flip((new Parser($firstQuery->getQuery()))
173+
->parse()->getResultSetMapping()->scalarMappings);
174+
$unionOfQueries = $unionOfQueries
175+
->addOrderBy($firstQueryFieldAliases['orderByField'], $isOrderByDesc === true ? 'DESC' : 'ASC')
176+
->addOrderBy($firstQueryFieldAliases['postId'])
177+
->setMaxResults($maxResults);
178+
$unionOfQueriesSQL = $unionOfQueries->getSQL();
179+
180+
$rsm = new ResultSetMapping();
181+
foreach ($firstQueryFieldAliases as $fieldName => $fieldAlias) {
182+
$rsm->addScalarResult($fieldAlias, $fieldName);
183+
}
184+
185+
['result' => $result, 'hasMorePages' => $hasMorePages, 'queryPlan' => $queryPlan] = $this->getQueryResult(
186+
$firstQuery->getEntityManager()->createNativeQuery($unionOfQueriesSQL, $rsm),
187+
$this->perPageItems
188+
);
189+
/** @var PostsKeyByTypePluralName $postsKeyByTypePluralName */
190+
$postsKeyByTypePluralName = $result
191+
->groupBy(static fn(/** @var UnionPostKey $unionPostKey */ array $unionPostKey) => $unionPostKey['postType'])
192+
->mapWithKeys(static fn(Collection $unionPostKeys, /** @var 'reply'|'subReply'|'thread' $postType */ string $postType) =>
193+
[Helper::POST_TYPE_TO_PLURAL[$postType] => $unionPostKeys
194+
->map(static function (/** @var UnionPostKey $unionPostKey */ array $unionPostKey) use ($postType) {
195+
[
196+
'postId' => $postId,
197+
'fid' => $fid,
198+
'tid' => $tid,
199+
'pid' => $pid,
200+
'orderByField' => $orderByFieldValue
201+
] = $unionPostKey;
202+
return match ($postType) {
203+
'thread' => new ThreadKey($fid, $postId, $orderByFieldValue),
204+
'reply' => new ReplyKey($fid, $tid, $postId, $orderByFieldValue),
205+
'subReply' => new SubReplyKey($fid, $tid, $pid, $postId, $orderByFieldValue)
206+
};
207+
})
208+
]);
209+
Helper::abortAPIIf(40401, $postsKeyByTypePluralName->every(static fn(Collection $i) => $i->isEmpty()));
210+
211+
return [
212+
'unionOfQueriesSQL' => $unionOfQueriesSQL,
213+
'postsKeyByTypePluralName' => $postsKeyByTypePluralName,
214+
'hasMorePages' => $hasMorePages,
215+
'queryPlan' => $queryPlan
216+
];
217+
}
151218
}

‎be/src/Repository/Post/PostRepository.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*/
1313
abstract class PostRepository extends BaseRepository
1414
{
15-
abstract public function selectPostKeyDTO(string $orderByField): QueryBuilder;
15+
abstract public function selectUnionPostKey(): QueryBuilder;
1616

1717
abstract public function getPosts(int $fid, array|\ArrayAccess $postsId): array;
1818
}

‎be/src/Repository/Post/ReplyRepository.php‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
namespace App\Repository\Post;
44

5-
use App\DTO\PostKey\Reply as ReplyKey;
65
use App\Entity\Post\Reply;
76
use Doctrine\ORM\QueryBuilder;
87
use Doctrine\Persistence\ManagerRegistry;
@@ -15,10 +14,10 @@ public function __construct(ManagerRegistry $registry)
1514
parent::__construct($registry, Reply::class);
1615
}
1716

18-
public function selectPostKeyDTO(string $orderByField): QueryBuilder
17+
public function selectUnionPostKey(): QueryBuilder
1918
{
2019
return $this->createQueryBuilder('t')
21-
->select('new ' . ReplyKey::class . "(t.fid, t.tid, t.pid, '$orderByField', t.$orderByField)");
20+
->select("'reply' AS postType", 't.pid AS postId', 't.fid', 't.tid', 't.pid');
2221
}
2322

2423
public function getPosts(int $fid, array|\ArrayAccess $postsId): array

0 commit comments

Comments
 (0)